From 74b20a3671c1745828f75f93ad36bad305904580 Mon Sep 17 00:00:00 2001 From: Bryn Cooke Date: Fri, 8 Apr 2022 15:55:59 +0200 Subject: [PATCH] Otel config (#782) * General restructring of Otel configuration ```yaml telemetry: apollo: endpoint: apollo_graph_ref: apollo_key: tracing: propagation: # Propagation is automatically enabled for any exporters that are enabled # but you can enable extras. This is mostly to support otlp. zipkin: true datadog: false trace_context: false jaeger: false baggage: false otlp: endpoint: Default protocol: Grpc http: .. grpc: .. zipkin: agent: endpoint: Default jaeger: agent: endpoint: Default datadog: endpoint: Default # Trace config is shared across all exporters trace_config: # Env variables can be used anywhere in any plugin config service_name: "${ENV_VARIABLE}" metrics: .. ``` Each of the exporters has its own source file. Other notes: * Otel provides integration with http for propagation. Removed our custom code and use the library. * Remove our propagation layer, otel provides propagators for each exporter. * Fix header extraction. Spans can now be collated from client to subgraphs. * Update supgergraph demo so that span propagation takes place * Unfied metrics and tracing config * Config enums changes to snake_case There is an issue round sampling that needs to be followed up on. Sampling happens at the pipeline level, so if the user enables sampling then discarded samples will not make it to spaceport. We need to discuss what we are going to do about this. Co-authored-by: bryn Co-authored-by: Geoffroy Couprie Co-authored-by: Coenen Benjamin --- .circleci/config.yml | 5 +- Cargo.lock | 107 +- NEXT_CHANGELOG.md | 154 +-- apollo-router-core/Cargo.toml | 1 + apollo-router-core/src/plugin.rs | 17 + .../src/services/tower_subgraph_service.rs | 27 +- apollo-router/Cargo.toml | 31 +- apollo-router/src/configuration/mod.rs | 70 - ...nfiguration__tests__schema_generation.snap | 1204 ++++++++--------- apollo-router/src/executable.rs | 10 - apollo-router/src/http_server_factory.rs | 2 +- apollo-router/src/layers/mod.rs | 1 - apollo-router/src/layers/opentracing.rs | 130 -- apollo-router/src/lib.rs | 7 +- apollo-router/src/plugins/telemetry/config.rs | 244 ++++ .../src/plugins/telemetry/metrics.rs | 307 ----- .../src/plugins/telemetry/metrics/mod.rs | 162 +++ .../src/plugins/telemetry/metrics/otlp.rs | 56 + .../plugins/telemetry/metrics/prometheus.rs | 72 + apollo-router/src/plugins/telemetry/mod.rs | 911 +++++-------- apollo-router/src/plugins/telemetry/otlp.rs | 237 ++++ .../src/plugins/telemetry/otlp/grpc.rs | 126 -- .../src/plugins/telemetry/otlp/http.rs | 31 - .../src/plugins/telemetry/otlp/mod.rs | 192 --- ..._serde__tests__serialize_metadata_map.snap | 9 + .../src/plugins/telemetry/tracing/apollo.rs | 43 + .../telemetry/tracing}/apollo_telemetry.rs | 20 +- .../src/plugins/telemetry/tracing/datadog.rs | 78 ++ .../src/plugins/telemetry/tracing/jaeger.rs | 109 ++ .../src/plugins/telemetry/tracing/mod.rs | 42 + .../src/plugins/telemetry/tracing/otlp.rs | 55 + ..._serde__tests__serialize_metadata_map.snap | 0 ..._serde__tests__serialize_metadata_map.snap | 10 + ..._serde__tests__serialize_metadata_map.snap | 9 + .../src/plugins/telemetry/tracing/zipkin.rs | 96 ++ apollo-router/src/router_factory.rs | 54 +- apollo-router/src/state_machine.rs | 14 +- apollo-router/src/warp_http_server_factory.rs | 52 +- apollo-spaceport/Cargo.toml | 1 + apollo-spaceport/README.md | 44 +- apollo-spaceport/src/server.rs | 39 +- apollo-spaceport/src/spaceport.rs | 2 +- dockerfiles/federation-demo/federation-demo | 2 +- docs/source/config.json | 6 +- .../source/configuration/apollo-telemetry.mdx | 49 + docs/source/configuration/metrics.mdx | 70 + docs/source/configuration/opentelemetry.mdx | 172 --- docs/source/configuration/overview.mdx | 17 + docs/source/configuration/spaceport.mdx | 72 - docs/source/configuration/tracing.mdx | 191 +++ examples/telemetry/README.md | 5 +- examples/telemetry/jaeger.router.yaml | 7 +- examples/telemetry/otlp.router.yaml | 9 +- examples/telemetry/spaceport.router.yaml | 14 - licenses.html | 251 +++- xtask/src/commands/test.rs | 14 +- 56 files changed, 3076 insertions(+), 2584 deletions(-) delete mode 100644 apollo-router/src/layers/mod.rs delete mode 100644 apollo-router/src/layers/opentracing.rs create mode 100644 apollo-router/src/plugins/telemetry/config.rs delete mode 100644 apollo-router/src/plugins/telemetry/metrics.rs create mode 100644 apollo-router/src/plugins/telemetry/metrics/mod.rs create mode 100644 apollo-router/src/plugins/telemetry/metrics/otlp.rs create mode 100644 apollo-router/src/plugins/telemetry/metrics/prometheus.rs create mode 100644 apollo-router/src/plugins/telemetry/otlp.rs delete mode 100644 apollo-router/src/plugins/telemetry/otlp/grpc.rs delete mode 100644 apollo-router/src/plugins/telemetry/otlp/http.rs delete mode 100644 apollo-router/src/plugins/telemetry/otlp/mod.rs create mode 100644 apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__otlp__metadata_map_serde__tests__serialize_metadata_map.snap create mode 100644 apollo-router/src/plugins/telemetry/tracing/apollo.rs rename apollo-router/src/{ => plugins/telemetry/tracing}/apollo_telemetry.rs (98%) create mode 100644 apollo-router/src/plugins/telemetry/tracing/datadog.rs create mode 100644 apollo-router/src/plugins/telemetry/tracing/jaeger.rs create mode 100644 apollo-router/src/plugins/telemetry/tracing/mod.rs create mode 100644 apollo-router/src/plugins/telemetry/tracing/otlp.rs rename apollo-router/src/plugins/telemetry/{otlp => tracing}/snapshots/apollo_router__plugins__telemetry__otlp__grpc__header_map_serde__tests__serialize_metadata_map.snap (100%) create mode 100644 apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__otlp__grpc__header_map_serde__tests__serialize_metadata_map.snap create mode 100644 apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__otlp__header_map_serde__tests__serialize_metadata_map.snap create mode 100644 apollo-router/src/plugins/telemetry/tracing/zipkin.rs create mode 100644 docs/source/configuration/apollo-telemetry.mdx create mode 100644 docs/source/configuration/metrics.mdx delete mode 100644 docs/source/configuration/opentelemetry.mdx delete mode 100644 docs/source/configuration/spaceport.mdx create mode 100644 docs/source/configuration/tracing.mdx delete mode 100644 examples/telemetry/spaceport.router.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ce627c1fd..8b9776d02a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -242,10 +242,7 @@ commands: steps: - rust/build: with_cache: false - crate: --locked --features otlp-grpc -p apollo-router -p apollo-router-core - - rust/build: - with_cache: false - crate: --locked --no-default-features --features otlp-http -p apollo-router -p apollo-router-core + crate: --locked -p apollo-router -p apollo-router-core build_all_permutations: steps: - build_common_permutations diff --git a/Cargo.lock b/Cargo.lock index d3491ac2f1..84aaaa6931 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -120,10 +120,12 @@ dependencies = [ "derive_more", "directories", "displaydoc", + "envmnt", "futures", "hotwatch", "http", "humantime", + "humantime-serde", "hyper", "insta", "itertools", @@ -132,9 +134,13 @@ dependencies = [ "mockall", "once_cell", "opentelemetry", + "opentelemetry-datadog", + "opentelemetry-http", "opentelemetry-jaeger", "opentelemetry-otlp", "opentelemetry-prometheus", + "opentelemetry-semantic-conventions", + "opentelemetry-zipkin", "prometheus", "prost-types", "regex", @@ -162,7 +168,7 @@ dependencies = [ "tracing-core", "tracing-opentelemetry", "tracing-subscriber", - "typed-builder", + "typed-builder 0.10.0", "url", "uuid", "warp", @@ -211,6 +217,7 @@ dependencies = [ "moka", "once_cell", "opentelemetry", + "opentelemetry-http", "paste", "regex", "router-bridge", @@ -234,7 +241,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", - "typed-builder", + "typed-builder 0.10.0", "url", "urlencoding", ] @@ -264,6 +271,7 @@ dependencies = [ "serde_json", "sys-info", "tokio", + "tokio-stream", "tonic", "tonic-build", "tracing", @@ -1169,6 +1177,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" +[[package]] +name = "dunce" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453440c271cf5577fd2a40e4942540cb7d0d2f85e27c8d07dd0023c925a67541" + [[package]] name = "dyn-clone" version = "1.0.4" @@ -1247,6 +1261,16 @@ dependencies = [ "termcolor", ] +[[package]] +name = "envmnt" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f96dd862f12fac698dec3932dff0e6fb34bffeb5515ae5932d620cfe076571e" +dependencies = [ + "fsio", + "indexmap", +] + [[package]] name = "error-chain" version = "0.12.4" @@ -1418,6 +1442,15 @@ dependencies = [ "libc", ] +[[package]] +name = "fsio" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e87827efaf94c7a44b562ff57de06930712fe21b530c3797cdede26e6377eb" +dependencies = [ + "dunce", +] + [[package]] name = "fslock" version = "0.1.8" @@ -1890,6 +1923,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime", + "serde", +] + [[package]] name = "hyper" version = "0.14.18" @@ -2737,6 +2780,25 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "opentelemetry-datadog" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "457462dc4cd365992c574c79181ff11ee6f66c5cbfb15a352217b4e0b35eac34" +dependencies = [ + "async-trait", + "http", + "indexmap", + "itertools", + "lazy_static", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-semantic-conventions", + "reqwest", + "rmp", + "thiserror", +] + [[package]] name = "opentelemetry-http" version = "0.6.0" @@ -2783,7 +2845,6 @@ dependencies = [ "opentelemetry-http", "prost", "prost-build", - "serde", "thiserror", "tokio", "tonic", @@ -2810,6 +2871,25 @@ dependencies = [ "opentelemetry", ] +[[package]] +name = "opentelemetry-zipkin" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb86c6e02de97a3a7ffa5d267e1ff3f0c930ccf8a31e286277a209af6ed6cfc1" +dependencies = [ + "async-trait", + "http", + "lazy_static", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-semantic-conventions", + "reqwest", + "serde", + "serde_json", + "thiserror", + "typed-builder 0.9.1", +] + [[package]] name = "ordered-float" version = "1.1.1" @@ -3453,6 +3533,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rmp" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f55e5fa1446c4d5dd1f5daeed2a4fe193071771a2636274d0d7a3b082aa7ad6" +dependencies = [ + "byteorder", + "num-traits", +] + [[package]] name = "router-bridge" version = "0.1.0" @@ -4694,6 +4784,17 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "typed-builder" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a46ee5bd706ff79131be9c94e7edcb82b703c487766a114434e5790361cf08c5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "typed-builder" version = "0.10.0" diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index f7f0a53631..13bb582cfc 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -20,125 +20,67 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - **Headline** ([PR #PR_NUMBER](https://github.com/apollographql/router/pull/PR_NUMBER)) Description! And a link to a [reference](http://url) - - --> - - -# [v0.1.0-preview.3] - 2022-04-08 -## 🚀 Features -- **Add version flag to router** ([PR #805](https://github.com/apollographql/router/pull/805)) - - You can now provider a `--version or -V` flag to the router. It will output version information and terminate. - -- **New startup message** ([PR #780](https://github.com/apollographql/router/pull/780)) - - The router startup message was updated with more links to documentation and version information. - -- **Add better support of introspection queries** ([PR #802](https://github.com/apollographql/router/pull/802)) - - Before this feature the Router didn't execute all the introspection queries, only a small number of the most common ones were executed. Now it detects if it's an introspection query, tries to fetch it from cache, if it's not in the cache we execute it and put the response in the cache. - -- **Add an option to disable the landing page** ([PR #801](https://github.com/apollographql/router/pull/801)) - - By default the router will display a landing page, which could be useful in development. If this is not - desirable the router can be configured to not display this landing page: - ```yaml - server: - landing_page: false - ``` - -- **Add support of metrics in `apollo.telemetry` plugin** ([PR #738](https://github.com/apollographql/router/pull/738)) - - The Router will now compute different metrics you can expose via Prometheus or OTLP exporter. +# [v0.1.0-preview.4] - Unreleased +## ❗ BREAKING ❗ +- **Telemetry simplification** [PR #782](https://github.com/apollographql/router/pull/782) - Example of configuration to export an endpoint (configured with the path `/plugins/apollo.telemetry/metrics`) with metrics in `Prometheus` format: + Telemetry configuration has been reworked to focus exporters rather than OpenTelemetry. Users can focus on what they are tying to integrate with rather than the fact that OpenTelemetry is used in the Apollo Router under the hood. ```yaml telemetry: + apollo: + endpoint: + apollo_graph_ref: + apollo_key: metrics: - exporter: - prometheus: - # By setting this endpoint you enable the prometheus exporter - # All our endpoints exposed by plugins are namespaced by the name of the plugin - # Then to access to this prometheus endpoint, the full url path will be `/plugins/apollo.telemetry/metrics` - endpoint: "/metrics" - ``` - -- **Add experimental support of `custom_endpoint` method in `Plugin` trait** ([PR #738](https://github.com/apollographql/router/pull/738)) - - The `custom_endpoint` method lets you declare a new endpoint exposed for your plugin. For now it's only accessible for official `apollo.` plugins and for `experimental.`. The return type of this method is a Tower [`Service`](). + prometheus: + enabled: true + tracing: + propagation: + # Propagation is automatically enabled for any exporters that are enabled, + # but you can enable extras. This is mostly to support otlp and opentracing. + zipkin: true + datadog: false + trace_context: false + jaeger: false + baggage: false -- **configurable subgraph error redaction** ([PR #797](https://github.com/apollographql/router/issues/797)) - By default, subgraph errors are not propagated to the user. This experimental plugin allows messages to be propagated either for all subgraphs or on - an individual subgraph basis. Individual subgraph configuration overrides the default (all) configuration. The configuration mechanism is similar - to that used in the `headers` plugin: - ```yaml - plugins: - experimental.include_subgraph_errors: - all: true + otlp: + endpoint: default + protocol: grpc + http: + .. + grpc: + .. + zipkin: + agent: + endpoint: default + jaeger: + agent: + endpoint: default + datadog: + endpoint: default ``` +## 🚀 Features +- **Datadog support** [PR #782](https://github.com/apollographql/router/pull/782) -- **Add a trace level log for subgraph queries** ([PR #808](https://github.com/apollographql/router/issues/808)) - - To debug the query plan execution, we added log messages to print the query plan, and for each subgraph query, - the operation, variables and response. It can be activated as follows: - - ``` - router -s supergraph.graphql --log info,apollo_router_core::query_planner::log=trace - ``` - -## 🐛 Fixes -- **Eliminate memory leaks when tasks are cancelled** [PR #758](https://github.com/apollographql/router/pull/758) - - The deduplication layer could leak memory when queries were cancelled and never retried: leaks were previously cleaned up on the next similar query. Now the leaking data will be deleted right when the query is cancelled - -- **Trim the query to better detect an empty query** ([PR #738](https://github.com/apollographql/router/pull/738)) - - Before this fix, if you wrote a query with only whitespaces inside, it wasn't detected as an empty query. - -- **Keep the original context in `RouterResponse` when returning an error** ([PR #738](https://github.com/apollographql/router/pull/738)) + Datadog support has been added via `telemetry` yaml configuration. - This fix keeps the original http request in `RouterResponse` when there is an error. +- **Yaml env variable expansion** [PR #782](https://github.com/apollographql/router/pull/782) -- **add a user-agent header to the studio usage ingress submission** ([PR #773](https://github.com/apollographql/router/pull/773)) + All values in the router configuration outside the `server` section may use environment variable expansion. + Unix style expansion is used. Either: - Requests to Studio now identify the router and its version + * `${ENV_VAR_NAME}`- Expands to the environment variable `ENV_VAR_NAME`. + * `${ENV_VAR_NAME:some_default}` - Expands to `ENV_VAR_NAME` or `some_default` if the environment variable did not exist. + Only values may be expanded (not keys): + ```yaml {4,8} title="router.yaml" + example: + passord: "${MY_PASSWORD}" + ``` +## 🐛 Fixes ## 🛠 Maintenance -- **A faster Query planner** ([PR #768](https://github.com/apollographql/router/pull/768)) - - We reworked the way query plans are generated before being cached, which lead to a great performance improvement. Moreover, the router is able to make sure the schema is valid at startup and on schema update, before you query it. - -- **Xtask improvements** ([PR #604](https://github.com/apollographql/router/pull/604)) - - The command we run locally to make sure tests, lints and compliance-checks pass will now edit the license file and run cargo fmt so you can directly commit it before you open a Pull Request - -- **Switch from reqwest to a Tower client for subgraph services** ([PR #769](https://github.com/apollographql/router/pull/769)) - - It results in better performance due to less URL parsing, and now header propagation falls under the apollo_router_core log filter, making it harder to disable accidentally - -- **Remove OpenSSL usage** ([PR #783](https://github.com/apollographql/router/pull/783) and [PR #810](https://github.com/apollographql/router/pull/810)) - - OpenSSL is used for HTTPS clients when connecting to subgraphs or the Studio API. It is now replaced with rustls, which is faster to compile and link - -- **Download the Studio protobuf schema during build** ([PR #776](https://github.com/apollographql/router/pull/776) - - The schema was vendored before, now it is downloaded dynamically during the build process - -- **Fix broken benchmarks** ([PR #797](https://github.com/apollographql/router/issues/797)) - - the `apollo-router-benchmarks` project was failing due to changes in the query planner. It is now fixed, and its subgraph mocking code is now available in `apollo-router-core` - ## 📚 Documentation - -- **Document the Plugin and DynPlugin trait** ([PR #800](https://github.com/apollographql/router/pull/800) - - Those traits are used to extend the router with Rust plugins \ No newline at end of file diff --git a/apollo-router-core/Cargo.toml b/apollo-router-core/Cargo.toml index 88f77ff29b..e8fc93efca 100644 --- a/apollo-router-core/Cargo.toml +++ b/apollo-router-core/Cargo.toml @@ -34,6 +34,7 @@ mockall = "0.11.0" moka = { version = "0.7.2", features = ["future", "futures-util"] } once_cell = "1.9.0" opentelemetry = "0.17.0" +opentelemetry-http = "0.6.0" paste = "1.0.6" regex = "1.5.5" router-bridge = { git = "https://github.com/apollographql/federation-rs.git", rev = "a1846dd1780989cbad767249c9c4f59a649523ad" } diff --git a/apollo-router-core/src/plugin.rs b/apollo-router-core/src/plugin.rs index 56822abafd..ffb9445e86 100644 --- a/apollo-router-core/src/plugin.rs +++ b/apollo-router-core/src/plugin.rs @@ -82,6 +82,12 @@ pub trait Plugin: Send + Sync + 'static + Sized { Ok(()) } + /// This method may be removed in future, do not use! + /// This is invoked after startup before a plugin goes live. + /// It must not panic. + #[deprecated] + fn activate(&mut self) {} + /// This is invoked whenever configuration is changed /// during router execution, including at shutdown. async fn shutdown(&mut self) -> Result<(), BoxError> { @@ -152,6 +158,12 @@ pub trait DynPlugin: Send + Sync + 'static { /// during router execution, including at startup. async fn startup(&mut self) -> Result<(), BoxError>; + /// This method may be removed in future, do not use! + /// This is invoked after startup before a plugin goes live. + /// It must not panic. + #[deprecated] + fn activate(&mut self); + /// This is invoked whenever configuration is changed /// during router execution, including at shutdown. async fn shutdown(&mut self) -> Result<(), BoxError>; @@ -205,6 +217,11 @@ where self.startup().await } + #[allow(deprecated)] + fn activate(&mut self) { + self.activate() + } + async fn shutdown(&mut self) -> Result<(), BoxError> { self.shutdown().await } diff --git a/apollo-router-core/src/services/tower_subgraph_service.rs b/apollo-router-core/src/services/tower_subgraph_service.rs index f885f77a82..71a93f0813 100644 --- a/apollo-router-core/src/services/tower_subgraph_service.rs +++ b/apollo-router-core/src/services/tower_subgraph_service.rs @@ -1,14 +1,15 @@ use crate::prelude::*; use futures::future::BoxFuture; +use global::get_text_map_propagator; use http::{ - header::{HeaderName, ACCEPT, CONTENT_TYPE}, + header::{ACCEPT, CONTENT_TYPE}, HeaderValue, }; use hyper::client::HttpConnector; use hyper_rustls::HttpsConnector; -use opentelemetry::{global, propagation::Injector}; +use opentelemetry::global; +use std::sync::Arc; use std::task::Poll; -use std::{str::FromStr, sync::Arc}; use tower::{BoxError, ServiceBuilder}; use tracing::{Instrument, Span}; use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -65,12 +66,10 @@ impl tower::Service for TowerSubgraphService { request.headers_mut().insert(CONTENT_TYPE, app_json.clone()); request.headers_mut().insert(ACCEPT, app_json); - global::get_text_map_propagator(|injector| { - injector.inject_context( + get_text_map_propagator(|propagator| { + propagator.inject_context( &Span::current().context(), - &mut RequestInjector { - request: &mut request, - }, + &mut opentelemetry_http::HeaderInjector(request.headers_mut()), ) }); @@ -112,15 +111,3 @@ impl tower::Service for TowerSubgraphService { }) } } - -struct RequestInjector<'a, T> { - request: &'a mut http::Request, -} - -impl<'a, T> Injector for RequestInjector<'a, T> { - fn set(&mut self, key: &str, value: String) { - let header_name = HeaderName::from_str(key).expect("Must be header name"); - let header_value = HeaderValue::from_str(&value).expect("Must be a header value"); - self.request.headers_mut().insert(header_name, header_value); - } -} diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index 0f23e341e2..41b0884086 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -10,19 +10,6 @@ publish = false name = "router" path = "src/main.rs" -[features] -default = ["otlp-grpc"] -otlp-grpc = [ - "opentelemetry-otlp/tonic", - "opentelemetry-otlp/tonic-build", - "opentelemetry-otlp/prost", - "opentelemetry-otlp/tls", - "tonic", - "tonic/transport", - "tonic/tls", -] -otlp-http = ["opentelemetry-otlp/http-proto"] - [dependencies] anyhow = "1.0.55" apollo-parser = { git = "https://github.com/apollographql/apollo-rs.git", rev = "e707e0f78f41ace1c3ecfe69bc10f4144ffbf7ac" } @@ -35,23 +22,33 @@ derivative = "2.2.0" derive_more = "0.99.17" directories = "4.0.1" displaydoc = "0.2" +envmnt = "0.9.1" futures = { version = "0.3.21", features = ["thread-pool"] } hotwatch = "0.4.6" http = "0.2.6" humantime = "2.1.0" +humantime-serde = "1.0.1" hyper = { version = "0.14.18", features = ["server"] } itertools = "0.10.3" once_cell = "1.9.0" opentelemetry = { version = "0.17.0", features = ["rt-tokio", "serialize", "metrics"] } +opentelemetry-datadog = {version="0.5.0", features=["reqwest-client"]} +opentelemetry-http = "0.6.0" opentelemetry-jaeger = { version = "0.16.0", features = [ "collector_client", "reqwest_collector_client", "rt-tokio", ] } opentelemetry-otlp = { version = "0.10.0", default-features = false, features = [ - "serialize", - "metrics" -], optional = true } + "tonic", + "tonic-build", + "prost", + "tls", + "http-proto", + "metrics", +] } +opentelemetry-semantic-conventions = "0.9.0" +opentelemetry-zipkin = "0.15.0" opentelemetry-prometheus = "0.10.0" prometheus = "0.13" prost-types = "0.9.0" @@ -68,7 +65,7 @@ clap = { version = "3.1.3", features = ["env", "derive"] } thiserror = "1.0.30" tokio = { version = "1.17.0", features = ["full"] } tokio-util = { version = "0.7.1", features = ["net", "codec"] } -tonic = { version = "0.6.2", optional = true } +tonic = { version = "0.6.2", features = ["transport", "tls"] } tower = { version = "0.4.12", features = ["full"] } tower-http = { version = "0.2.5", features = ["trace"] } tower-service = "0.3.1" diff --git a/apollo-router/src/configuration/mod.rs b/apollo-router/src/configuration/mod.rs index f9798ceb97..31f98865aa 100644 --- a/apollo-router/src/configuration/mod.rs +++ b/apollo-router/src/configuration/mod.rs @@ -367,68 +367,6 @@ impl Cors { } } -pub(crate) fn default_service_name() -> String { - "router".to_string() -} - -pub(crate) fn default_service_namespace() -> String { - "apollo".to_string() -} - -/// Types of secret. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields, rename_all = "snake_case")] -pub enum Secret { - Env(String), - File(PathBuf), -} - -impl Secret { - pub fn read(&self) -> Result { - match self { - Secret::Env(s) => std::env::var(s).map_err(ConfigurationError::CannotReadSecretFromEnv), - Secret::File(path) => { - std::fs::read_to_string(path).map_err(ConfigurationError::CannotReadSecretFromFile) - } - } - } -} - -/// TLS configuration details. -#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct TlsConfig { - domain_name: Option, - ca: Option, - cert: Option, - key: Option, -} - -#[cfg(feature = "otlp-grpc")] -impl TlsConfig { - pub fn tls_config( - &self, - ) -> Result { - let mut config = tonic::transport::channel::ClientTlsConfig::new(); - - if let Some(domain_name) = self.domain_name.as_ref() { - config = config.domain_name(domain_name); - } - - if let Some(ca_certificate) = self.ca.as_ref() { - let certificate = tonic::transport::Certificate::from_pem(ca_certificate.read()?); - config = config.ca_certificate(certificate); - } - - if let (Some(cert), Some(key)) = (self.cert.as_ref(), self.key.as_ref()) { - let identity = tonic::transport::Identity::from_pem(cert.read()?, key.read()?); - config = config.identity(identity); - } - - Ok(config) - } -} - #[cfg(test)] mod tests { use super::*; @@ -436,10 +374,8 @@ mod tests { use apollo_router_core::SchemaError; use http::Uri; #[cfg(unix)] - #[cfg(any(feature = "otlp-grpc"))] use insta::assert_json_snapshot; #[cfg(unix)] - #[cfg(any(feature = "otlp-grpc"))] use schemars::gen::SchemaSettings; use std::collections::HashMap; @@ -453,7 +389,6 @@ mod tests { } #[cfg(unix)] - #[cfg(any(feature = "otlp-grpc"))] #[test] fn schema_generation() { let settings = SchemaSettings::draft2019_09().with(|s| { @@ -479,33 +414,28 @@ mod tests { assert_config_snapshot!("testdata/config_opentelemetry_jaeger_full.yml"); } - #[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] #[test] fn ensure_configuration_api_does_not_change_common() { // NOTE: don't take a snapshot here because the optional fields appear with ~ and they vary // per implementation - #[cfg(feature = "otlp-http")] serde_yaml::from_str::(include_str!( "testdata/config_opentelemetry_otlp_tracing_http_common.yml" )) .unwrap(); - #[cfg(feature = "otlp-grpc")] serde_yaml::from_str::(include_str!( "testdata/config_opentelemetry_otlp_tracing_grpc_common.yml" )) .unwrap(); } - #[cfg(feature = "otlp-grpc")] #[test] fn ensure_configuration_api_does_not_change_grpc() { assert_config_snapshot!("testdata/config_opentelemetry_otlp_tracing_grpc_basic.yml"); assert_config_snapshot!("testdata/config_opentelemetry_otlp_tracing_grpc_full.yml"); } - #[cfg(feature = "otlp-http")] #[test] fn ensure_configuration_api_does_not_change_http() { assert_config_snapshot!("testdata/config_opentelemetry_otlp_tracing_http_basic.yml"); diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index 2887ab4a22..4564b7c661 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -418,30 +418,286 @@ expression: "&schema" "telemetry": { "type": "object", "properties": { + "apollo": { + "type": "object", + "properties": { + "apollo_graph_ref": { + "type": "string", + "nullable": true + }, + "apollo_key": { + "type": "string", + "nullable": true + }, + "endpoint": { + "type": "string", + "format": "uri", + "nullable": true + } + }, + "additionalProperties": false, + "nullable": true + }, "metrics": { "type": "object", - "required": [ - "exporter" - ], "properties": { - "exporter": { + "common": { + "type": "object", + "required": [ + "delay_interval" + ], + "properties": { + "delay_interval": { + "type": "object", + "required": [ + "nanos", + "secs" + ], + "properties": { + "nanos": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "secs": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + } + } + } + }, + "additionalProperties": false, + "nullable": true + }, + "otlp": { + "type": "object", + "required": [ + "endpoint" + ], + "properties": { + "endpoint": { + "anyOf": [ + { + "type": "string", + "enum": [ + "default" + ] + }, + { + "type": "string", + "format": "uri" + } + ] + }, + "grpc": { + "type": "object", + "properties": { + "ca": { + "oneOf": [ + { + "type": "object", + "required": [ + "env" + ], + "properties": { + "env": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string" + } + }, + "additionalProperties": false + } + ], + "nullable": true + }, + "cert": { + "oneOf": [ + { + "type": "object", + "required": [ + "env" + ], + "properties": { + "env": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string" + } + }, + "additionalProperties": false + } + ], + "nullable": true + }, + "domain_name": { + "type": "string", + "nullable": true + }, + "key": { + "oneOf": [ + { + "type": "object", + "required": [ + "env" + ], + "properties": { + "env": { + "type": "string" + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string" + } + }, + "additionalProperties": false + } + ], + "nullable": true + }, + "metadata": { + "default": null, + "type": "object", + "additionalProperties": true, + "nullable": true + } + }, + "additionalProperties": false, + "nullable": true + }, + "http": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "nullable": true + } + }, + "additionalProperties": false, + "nullable": true + }, + "protocol": { + "type": "string", + "enum": [ + "grpc", + "http" + ], + "nullable": true + }, + "timeout": { + "default": null, + "type": "string" + } + }, + "additionalProperties": false, + "nullable": true + }, + "prometheus": { + "type": "object", + "required": [ + "enabled" + ], + "properties": { + "enabled": { + "type": "boolean" + } + }, + "additionalProperties": false, + "nullable": true + } + }, + "additionalProperties": false, + "nullable": true + }, + "tracing": { + "type": "object", + "properties": { + "datadog": { + "type": "object", + "required": [ + "endpoint" + ], + "properties": { + "endpoint": { + "anyOf": [ + { + "type": "string", + "enum": [ + "default" + ] + }, + { + "type": "string", + "format": "uri" + } + ] + } + }, + "additionalProperties": false, + "nullable": true + }, + "jaeger": { + "type": "object", "oneOf": [ { "type": "object", "required": [ - "prometheus" + "agent" ], "properties": { - "prometheus": { + "agent": { "type": "object", "required": [ "endpoint" ], "properties": { "endpoint": { - "type": "string" + "anyOf": [ + { + "type": "string", + "enum": [ + "default" + ] + }, + { + "type": "string" + } + ] } - } + }, + "additionalProperties": false } }, "additionalProperties": false @@ -449,187 +705,69 @@ expression: "&schema" { "type": "object", "required": [ - "otlp" + "collector" ], "properties": { - "otlp": { + "collector": { "type": "object", - "oneOf": [ - { - "type": "object", - "required": [ - "grpc" - ], - "properties": { - "grpc": { - "type": "object", - "properties": { - "endpoint": { - "default": null, - "type": "string", - "format": "uri", - "nullable": true - }, - "metadata": { - "default": null, - "type": "object", - "additionalProperties": true, - "nullable": true - }, - "protocol": { - "default": null, - "type": "string", - "enum": [ - "Grpc", - "HttpBinary" - ], - "nullable": true - }, - "timeout": { - "type": "integer", - "format": "uint64", - "minimum": 0.0, - "nullable": true - }, - "tls_config": { - "description": "TLS configuration details.", - "type": "object", - "properties": { - "ca": { - "description": "Types of secret.", - "oneOf": [ - { - "type": "object", - "required": [ - "env" - ], - "properties": { - "env": { - "type": "string" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "file" - ], - "properties": { - "file": { - "type": "string" - } - }, - "additionalProperties": false - } - ], - "nullable": true - }, - "cert": { - "description": "Types of secret.", - "oneOf": [ - { - "type": "object", - "required": [ - "env" - ], - "properties": { - "env": { - "type": "string" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "file" - ], - "properties": { - "file": { - "type": "string" - } - }, - "additionalProperties": false - } - ], - "nullable": true - }, - "domain_name": { - "type": "string", - "nullable": true - }, - "key": { - "description": "Types of secret.", - "oneOf": [ - { - "type": "object", - "required": [ - "env" - ], - "properties": { - "env": { - "type": "string" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "file" - ], - "properties": { - "file": { - "type": "string" - } - }, - "additionalProperties": false - } - ], - "nullable": true - } - }, - "additionalProperties": false, - "nullable": true - } - }, - "additionalProperties": false, - "nullable": true - } - }, - "additionalProperties": false + "required": [ + "endpoint" + ], + "properties": { + "endpoint": { + "type": "string", + "format": "uri" + }, + "password": { + "type": "string", + "nullable": true + }, + "username": { + "type": "string", + "nullable": true } - ] + }, + "additionalProperties": false } }, "additionalProperties": false } - ] - } - }, - "nullable": true - }, - "opentelemetry": { - "oneOf": [ - { + ], + "additionalProperties": false, + "nullable": true + }, + "otlp": { "type": "object", "required": [ - "jaeger" + "endpoint" ], "properties": { - "jaeger": { + "endpoint": { + "anyOf": [ + { + "type": "string", + "enum": [ + "default" + ] + }, + { + "type": "string", + "format": "uri" + } + ] + }, + "grpc": { "type": "object", "properties": { - "endpoint": { + "ca": { "oneOf": [ { "type": "object", "required": [ - "agent" + "env" ], "properties": { - "agent": { + "env": { "type": "string" } }, @@ -638,12 +776,11 @@ expression: "&schema" { "type": "object", "required": [ - "collector" + "file" ], "properties": { - "collector": { - "type": "string", - "format": "uri" + "file": { + "type": "string" } }, "additionalProperties": false @@ -651,476 +788,321 @@ expression: "&schema" ], "nullable": true }, - "service_name": { - "default": "router", - "type": "string" - }, - "trace_config": { - "type": "object", - "properties": { - "attributes": { + "cert": { + "oneOf": [ + { "type": "object", - "additionalProperties": { - "anyOf": [ - { - "description": "bool values", - "type": "boolean" - }, - { - "description": "i64 values", - "type": "integer", - "format": "int64" - }, - { - "description": "f64 values", - "type": "number", - "format": "double" - }, - { - "description": "String values", - "type": "string" - }, - { - "description": "Array of homogeneous values", - "anyOf": [ - { - "description": "Array of bools", - "type": "array", - "items": { - "type": "boolean" - } - }, - { - "description": "Array of integers", - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - }, - { - "description": "Array of floats", - "type": "array", - "items": { - "type": "number", - "format": "double" - } - }, - { - "description": "Array of strings", - "type": "array", - "items": { - "type": "string" - } - } - ] - } - ] + "required": [ + "env" + ], + "properties": { + "env": { + "type": "string" + } }, - "nullable": true - }, - "max_attributes_per_event": { - "type": "integer", - "format": "uint32", - "minimum": 0.0, - "nullable": true - }, - "max_attributes_per_link": { - "type": "integer", - "format": "uint32", - "minimum": 0.0, - "nullable": true - }, - "max_attributes_per_span": { - "type": "integer", - "format": "uint32", - "minimum": 0.0, - "nullable": true - }, - "max_events_per_span": { - "type": "integer", - "format": "uint32", - "minimum": 0.0, - "nullable": true - }, - "max_links_per_span": { - "type": "integer", - "format": "uint32", - "minimum": 0.0, - "nullable": true + "additionalProperties": false }, - "sampler": { - "default": null, - "oneOf": [ - { - "type": "string", - "enum": [ - "AlwaysOn", - "AlwaysOff" - ] - }, - { - "description": "Respects the parent span's sampling decision or delegates a delegate sampler for root spans. Not supported via yaml config Sample a given fraction of traces. Fractions >= 1 will always sample. If the parent span is sampled, then it's child spans will automatically be sampled. Fractions < 0 are treated as zero, but spans may still be sampled if their parent is.", - "type": "object", - "required": [ - "TraceIdRatioBased" - ], - "properties": { - "TraceIdRatioBased": { - "type": "number", - "format": "double" - } - }, - "additionalProperties": false + { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string" + } + }, + "additionalProperties": false + } + ], + "nullable": true + }, + "domain_name": { + "type": "string", + "nullable": true + }, + "key": { + "oneOf": [ + { + "type": "object", + "required": [ + "env" + ], + "properties": { + "env": { + "type": "string" } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "file" ], - "nullable": true + "properties": { + "file": { + "type": "string" + } + }, + "additionalProperties": false } + ], + "nullable": true + }, + "metadata": { + "default": null, + "type": "object", + "additionalProperties": true, + "nullable": true + } + }, + "additionalProperties": false, + "nullable": true + }, + "http": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" }, - "additionalProperties": false, "nullable": true } }, "additionalProperties": false, "nullable": true + }, + "protocol": { + "type": "string", + "enum": [ + "grpc", + "http" + ], + "nullable": true + }, + "timeout": { + "default": null, + "type": "string" } }, - "additionalProperties": false + "additionalProperties": false, + "nullable": true }, - { + "propagation": { "type": "object", - "required": [ - "otlp" - ], "properties": { - "otlp": { + "baggage": { + "type": "boolean", + "nullable": true + }, + "datadog": { + "type": "boolean", + "nullable": true + }, + "jaeger": { + "type": "boolean", + "nullable": true + }, + "trace_context": { + "type": "boolean", + "nullable": true + }, + "zipkin": { + "type": "boolean", + "nullable": true + } + }, + "additionalProperties": false, + "nullable": true + }, + "trace_config": { + "type": "object", + "properties": { + "attributes": { "type": "object", + "additionalProperties": { + "anyOf": [ + { + "description": "bool values", + "type": "boolean" + }, + { + "description": "i64 values", + "type": "integer", + "format": "int64" + }, + { + "description": "f64 values", + "type": "number", + "format": "double" + }, + { + "description": "String values", + "type": "string" + }, + { + "description": "Array of homogeneous values", + "anyOf": [ + { + "description": "Array of bools", + "type": "array", + "items": { + "type": "boolean" + } + }, + { + "description": "Array of integers", + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + { + "description": "Array of floats", + "type": "array", + "items": { + "type": "number", + "format": "double" + } + }, + { + "description": "Array of strings", + "type": "array", + "items": { + "type": "string" + } + } + ] + } + ] + }, + "nullable": true + }, + "max_attributes_per_event": { + "type": "integer", + "format": "uint32", + "minimum": 0.0, + "nullable": true + }, + "max_attributes_per_link": { + "type": "integer", + "format": "uint32", + "minimum": 0.0, + "nullable": true + }, + "max_attributes_per_span": { + "type": "integer", + "format": "uint32", + "minimum": 0.0, + "nullable": true + }, + "max_events_per_span": { + "type": "integer", + "format": "uint32", + "minimum": 0.0, + "nullable": true + }, + "max_links_per_span": { + "type": "integer", + "format": "uint32", + "minimum": 0.0, + "nullable": true + }, + "parent_based_sampler": { + "type": "boolean", + "nullable": true + }, + "sampler": { + "anyOf": [ + { + "description": "Sample a given fraction of traces. Fractions >= 1 will always sample. If the parent span is sampled, then it's child spans will automatically be sampled. Fractions < 0 are treated as zero, but spans may still be sampled if their parent is.", + "type": "number", + "format": "double" + }, + { + "type": "string", + "enum": [ + "always_on", + "always_off" + ] + } + ], + "nullable": true + }, + "service_name": { + "type": "string", + "nullable": true + }, + "service_namespace": { + "type": "string", + "nullable": true + } + }, + "additionalProperties": false, + "nullable": true + }, + "zipkin": { + "type": "object", + "oneOf": [ + { + "type": "object", + "required": [ + "agent" + ], "properties": { - "tracing": { + "agent": { "type": "object", "required": [ - "exporter" + "endpoint" ], "properties": { - "exporter": { - "oneOf": [ + "endpoint": { + "anyOf": [ { - "type": "object", - "required": [ - "grpc" - ], - "properties": { - "grpc": { - "type": "object", - "properties": { - "endpoint": { - "default": null, - "type": "string", - "format": "uri", - "nullable": true - }, - "metadata": { - "default": null, - "type": "object", - "additionalProperties": true, - "nullable": true - }, - "protocol": { - "default": null, - "type": "string", - "enum": [ - "Grpc", - "HttpBinary" - ], - "nullable": true - }, - "timeout": { - "type": "integer", - "format": "uint64", - "minimum": 0.0, - "nullable": true - }, - "tls_config": { - "description": "TLS configuration details.", - "type": "object", - "properties": { - "ca": { - "description": "Types of secret.", - "oneOf": [ - { - "type": "object", - "required": [ - "env" - ], - "properties": { - "env": { - "type": "string" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "file" - ], - "properties": { - "file": { - "type": "string" - } - }, - "additionalProperties": false - } - ], - "nullable": true - }, - "cert": { - "description": "Types of secret.", - "oneOf": [ - { - "type": "object", - "required": [ - "env" - ], - "properties": { - "env": { - "type": "string" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "file" - ], - "properties": { - "file": { - "type": "string" - } - }, - "additionalProperties": false - } - ], - "nullable": true - }, - "domain_name": { - "type": "string", - "nullable": true - }, - "key": { - "description": "Types of secret.", - "oneOf": [ - { - "type": "object", - "required": [ - "env" - ], - "properties": { - "env": { - "type": "string" - } - }, - "additionalProperties": false - }, - { - "type": "object", - "required": [ - "file" - ], - "properties": { - "file": { - "type": "string" - } - }, - "additionalProperties": false - } - ], - "nullable": true - } - }, - "additionalProperties": false, - "nullable": true - } - }, - "additionalProperties": false, - "nullable": true - } - }, - "additionalProperties": false - } - ] - }, - "trace_config": { - "type": "object", - "properties": { - "attributes": { - "type": "object", - "additionalProperties": { - "anyOf": [ - { - "description": "bool values", - "type": "boolean" - }, - { - "description": "i64 values", - "type": "integer", - "format": "int64" - }, - { - "description": "f64 values", - "type": "number", - "format": "double" - }, - { - "description": "String values", - "type": "string" - }, - { - "description": "Array of homogeneous values", - "anyOf": [ - { - "description": "Array of bools", - "type": "array", - "items": { - "type": "boolean" - } - }, - { - "description": "Array of integers", - "type": "array", - "items": { - "type": "integer", - "format": "int64" - } - }, - { - "description": "Array of floats", - "type": "array", - "items": { - "type": "number", - "format": "double" - } - }, - { - "description": "Array of strings", - "type": "array", - "items": { - "type": "string" - } - } - ] - } - ] - }, - "nullable": true - }, - "max_attributes_per_event": { - "type": "integer", - "format": "uint32", - "minimum": 0.0, - "nullable": true - }, - "max_attributes_per_link": { - "type": "integer", - "format": "uint32", - "minimum": 0.0, - "nullable": true - }, - "max_attributes_per_span": { - "type": "integer", - "format": "uint32", - "minimum": 0.0, - "nullable": true - }, - "max_events_per_span": { - "type": "integer", - "format": "uint32", - "minimum": 0.0, - "nullable": true - }, - "max_links_per_span": { - "type": "integer", - "format": "uint32", - "minimum": 0.0, - "nullable": true + "type": "string", + "enum": [ + "default" + ] }, - "sampler": { - "default": null, - "oneOf": [ - { - "type": "string", - "enum": [ - "AlwaysOn", - "AlwaysOff" - ] - }, - { - "description": "Respects the parent span's sampling decision or delegates a delegate sampler for root spans. Not supported via yaml config Sample a given fraction of traces. Fractions >= 1 will always sample. If the parent span is sampled, then it's child spans will automatically be sampled. Fractions < 0 are treated as zero, but spans may still be sampled if their parent is.", - "type": "object", - "required": [ - "TraceIdRatioBased" - ], - "properties": { - "TraceIdRatioBased": { - "type": "number", - "format": "double" - } - }, - "additionalProperties": false - } - ], - "nullable": true + { + "type": "string" } - }, - "additionalProperties": false, - "nullable": true + ] } }, - "additionalProperties": false, - "nullable": true + "additionalProperties": false + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "collector" + ], + "properties": { + "collector": { + "type": "object", + "required": [ + "endpoint" + ], + "properties": { + "endpoint": { + "type": "string", + "format": "uri" + } + }, + "additionalProperties": false } }, "additionalProperties": false } - }, - "additionalProperties": false - } - ], - "nullable": true - }, - "opentracing": { - "type": "object", - "required": [ - "format" - ], - "properties": { - "format": { - "type": "string", - "enum": [ - "jaeger", - "zipkin_b3" - ] - } - }, - "nullable": true - }, - "spaceport": { - "type": "object", - "required": [ - "external" - ], - "properties": { - "collector": { - "default": "https://127.0.0.1:50051", - "type": "string" - }, - "external": { - "type": "boolean" - }, - "listener": { - "default": "127.0.0.1:50051", - "type": "string" + ], + "additionalProperties": false, + "nullable": true } }, "additionalProperties": false, diff --git a/apollo-router/src/executable.rs b/apollo-router/src/executable.rs index 673623b5af..7fbc8bf3b1 100644 --- a/apollo-router/src/executable.rs +++ b/apollo-router/src/executable.rs @@ -266,16 +266,6 @@ APOLLO ROUTER v{version} } }; - // Create your text map propagator & assign it as the global propagator. - // - // This is required in order to create the header traceparent used in http_subgraph to - // propagate the trace id to the subgraph services. - // - // /!\ If this is not called, there will be no warning and no header will be sent to the - // subgraphs! - let propagator = opentelemetry::sdk::propagation::TraceContextPropagator::new(); - opentelemetry::global::set_text_map_propagator(propagator); - let server = ApolloRouterBuilder::default() .configuration(configuration) .schema(schema) diff --git a/apollo-router/src/http_server_factory.rs b/apollo-router/src/http_server_factory.rs index a1213bc845..6d1013c1bb 100644 --- a/apollo-router/src/http_server_factory.rs +++ b/apollo-router/src/http_server_factory.rs @@ -108,7 +108,7 @@ impl HttpServerHandle { // it is necessary to keep the queue of new TCP sockets associated with // the listener instead of dropping them let listener = self.server_future.await; - tracing::info!("previous server is closed"); + tracing::debug!("previous server stopped"); // we keep the TCP listener if it is compatible with the new configuration let listener = if self.listen_address != configuration.server.listen { diff --git a/apollo-router/src/layers/mod.rs b/apollo-router/src/layers/mod.rs deleted file mode 100644 index 364a03a430..0000000000 --- a/apollo-router/src/layers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod opentracing; diff --git a/apollo-router/src/layers/opentracing.rs b/apollo-router/src/layers/opentracing.rs deleted file mode 100644 index faf0cb496c..0000000000 --- a/apollo-router/src/layers/opentracing.rs +++ /dev/null @@ -1,130 +0,0 @@ -use std::fmt::Display; -use std::task::{Context, Poll}; - -use apollo_router_core::SubgraphRequest; -use http::HeaderValue; -use opentelemetry::trace::TraceContextExt; -use schemars::JsonSchema; -use serde::Deserialize; -use tower::{Layer, Service}; -use tracing::instrument::Instrumented; -use tracing::{span, Instrument, Level, Span}; -use tracing_opentelemetry::OpenTelemetrySpanExt; - -#[derive(Clone, JsonSchema, Deserialize, Debug)] -#[serde(rename_all = "snake_case")] -pub enum PropagationFormat { - Jaeger, - ZipkinB3, -} - -impl Display for PropagationFormat { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - PropagationFormat::Jaeger => write!(f, "jaeger"), - PropagationFormat::ZipkinB3 => write!(f, "zipkin_b3"), - } - } -} - -#[derive(Clone, JsonSchema, Deserialize, Debug)] -pub struct OpenTracingConfig { - format: PropagationFormat, -} - -#[derive(Debug)] -pub struct OpenTracingLayer { - format: PropagationFormat, -} - -impl OpenTracingLayer { - pub(crate) fn new(config: OpenTracingConfig) -> Self { - Self { - format: config.format, - } - } -} - -impl Layer for OpenTracingLayer { - type Service = OpenTracingService; - - fn layer(&self, inner: S) -> Self::Service { - OpenTracingService { - inner, - format: self.format.clone(), - } - } -} - -pub struct OpenTracingService { - inner: S, - format: PropagationFormat, -} - -impl Service for OpenTracingService -where - S: Service, -{ - type Response = S::Response; - type Error = S::Error; - type Future = Instrumented<>::Future>; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.inner.poll_ready(cx) - } - - fn call(&mut self, mut req: SubgraphRequest) -> Self::Future { - let current_span = Span::current(); - let span_context = current_span.context(); - let span_ref = span_context.span(); - let current_span_ctx = span_ref.span_context(); - let (trace_id, parent_span_id, trace_flags) = ( - current_span_ctx.trace_id(), - current_span_ctx.span_id(), - current_span_ctx.trace_flags(), - ); - - let new_span = span!(parent: current_span, Level::TRACE, "subgraph_request"); - let new_span_context = new_span.context(); - let new_span_ref = new_span_context.span(); - let span_id = new_span_ref.span_context().span_id(); - - match self.format { - PropagationFormat::Jaeger => { - req.http_request.headers_mut().insert( - "uber-trace-id", - HeaderValue::from_str(&format!( - "{}:{}:{}:{}", - trace_id, - parent_span_id, - span_id, - trace_flags.to_u8() - )) - .unwrap(), - ); - } - PropagationFormat::ZipkinB3 => { - req.http_request.headers_mut().insert( - "X-B3-TraceId", - HeaderValue::from_str(&trace_id.to_string()).unwrap(), - ); - req.http_request.headers_mut().insert( - "X-B3-SpanId", - HeaderValue::from_str(&span_id.to_string()).unwrap(), - ); - req.http_request.headers_mut().insert( - "X-B3-ParentSpanId", - HeaderValue::from_str(&parent_span_id.to_string()).unwrap(), - ); - req.http_request.headers_mut().insert( - "X-B3-Sampled", - HeaderValue::from_static( - current_span_ctx.is_sampled().then(|| "1").unwrap_or("0"), - ), - ); - } - } - - self.inner.call(req).instrument(new_span) - } -} diff --git a/apollo-router/src/lib.rs b/apollo-router/src/lib.rs index b46875c11b..ad489df824 100644 --- a/apollo-router/src/lib.rs +++ b/apollo-router/src/lib.rs @@ -1,11 +1,9 @@ //! Starts a server that will handle http graphql requests. -mod apollo_telemetry; pub mod configuration; mod executable; mod files; mod http_server_factory; -mod layers; pub mod plugins; mod reload; mod router_factory; @@ -13,7 +11,6 @@ mod state_machine; pub mod subscriber; mod warp_http_server_factory; -use crate::apollo_telemetry::SpaceportConfig; use crate::reload::Error as ReloadError; use crate::router_factory::{RouterServiceFactory, YamlRouterServiceFactory}; use crate::state_machine::StateMachine; @@ -71,8 +68,8 @@ pub enum FederatedServerError { /// could not create the HTTP server: {0} ServerCreationError(std::io::Error), - /// could not configure spaceport: {0} - ServerSpaceportError(tokio::sync::mpsc::error::SendError), + /// could not configure spaceport + ServerSpaceportError, /// no reload handle available NoReloadTracingHandleError, diff --git a/apollo-router/src/plugins/telemetry/config.rs b/apollo-router/src/plugins/telemetry/config.rs new file mode 100644 index 0000000000..3f646ebdb0 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/config.rs @@ -0,0 +1,244 @@ +use super::*; +use crate::plugins::telemetry::metrics; +use opentelemetry::sdk::Resource; +use opentelemetry::{Array, KeyValue, Value}; +use schemars::JsonSchema; +use serde::Deserialize; +use std::borrow::Cow; +use std::collections::BTreeMap; +use std::time::Duration; + +pub trait GenericWith +where + Self: Sized, +{ + fn with(self, option: &Option, apply: fn(Self, &B) -> Self) -> Self { + if let Some(option) = option { + return apply(self, option); + } + self + } + fn try_with( + self, + option: &Option, + apply: fn(Self, &B) -> Result, + ) -> Result { + if let Some(option) = option { + return apply(self, option); + } + Ok(self) + } +} + +impl GenericWith for T where Self: Sized {} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub struct Conf { + #[allow(dead_code)] + pub metrics: Option, + pub tracing: Option, + pub apollo: Option, +} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +#[allow(dead_code)] +pub struct Metrics { + pub common: Option, + pub otlp: Option, + pub prometheus: Option, +} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub struct MetricsCommon { + pub delay_interval: Duration, +} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub struct Tracing { + pub propagation: Option, + pub trace_config: Option, + pub otlp: Option, + pub jaeger: Option, + pub zipkin: Option, + pub datadog: Option, +} + +#[derive(Clone, Default, Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub struct Propagation { + pub baggage: Option, + pub trace_context: Option, + pub jaeger: Option, + pub datadog: Option, + pub zipkin: Option, +} + +#[derive(Default, Debug, Clone, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Trace { + pub service_name: Option, + pub service_namespace: Option, + pub sampler: Option, + pub parent_based_sampler: Option, + pub max_events_per_span: Option, + pub max_attributes_per_span: Option, + pub max_links_per_span: Option, + pub max_attributes_per_event: Option, + pub max_attributes_per_link: Option, + pub attributes: Option>, +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +#[serde(untagged, deny_unknown_fields)] +pub enum AttributeValue { + /// bool values + Bool(bool), + /// i64 values + I64(i64), + /// f64 values + F64(f64), + /// String values + String(String), + /// Array of homogeneous values + Array(AttributeArray), +} + +impl From for opentelemetry::Value { + fn from(value: AttributeValue) -> Self { + match value { + AttributeValue::Bool(v) => Value::Bool(v), + AttributeValue::I64(v) => Value::I64(v), + AttributeValue::F64(v) => Value::F64(v), + AttributeValue::String(v) => Value::String(Cow::from(v)), + AttributeValue::Array(v) => Value::Array(v.into()), + } + } +} + +#[derive(Debug, Clone, Deserialize, JsonSchema)] +#[serde(untagged, deny_unknown_fields)] +pub enum AttributeArray { + /// Array of bools + Bool(Vec), + /// Array of integers + I64(Vec), + /// Array of floats + F64(Vec), + /// Array of strings + String(Vec>), +} + +impl From for opentelemetry::Array { + fn from(array: AttributeArray) -> Self { + match array { + AttributeArray::Bool(v) => Array::Bool(v), + AttributeArray::I64(v) => Array::I64(v), + AttributeArray::F64(v) => Array::F64(v), + AttributeArray::String(v) => Array::String(v), + } + } +} + +#[derive(Clone, Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, untagged)] +pub enum SamplerOption { + /// Sample a given fraction of traces. Fractions >= 1 will always sample. If the parent span is + /// sampled, then it's child spans will automatically be sampled. Fractions < 0 are treated as + /// zero, but spans may still be sampled if their parent is. + TraceIdRatioBased(f64), + Always(Sampler), +} + +#[derive(Clone, Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum Sampler { + /// Always sample the trace + AlwaysOn, + /// Never sample the trace + AlwaysOff, +} + +impl From<&Trace> for opentelemetry::sdk::trace::Config { + fn from(config: &Trace) -> Self { + let mut trace_config = opentelemetry::sdk::trace::config(); + + let sampler = match (&config.sampler, &config.parent_based_sampler) { + (Some(SamplerOption::Always(Sampler::AlwaysOn)), Some(true)) => { + Some(parent_based(opentelemetry::sdk::trace::Sampler::AlwaysOn)) + } + (Some(SamplerOption::Always(Sampler::AlwaysOff)), Some(true)) => { + Some(parent_based(opentelemetry::sdk::trace::Sampler::AlwaysOff)) + } + (Some(SamplerOption::TraceIdRatioBased(ratio)), Some(true)) => Some(parent_based( + opentelemetry::sdk::trace::Sampler::TraceIdRatioBased(*ratio), + )), + (Some(SamplerOption::Always(Sampler::AlwaysOn)), _) => { + Some(opentelemetry::sdk::trace::Sampler::AlwaysOn) + } + (Some(SamplerOption::Always(Sampler::AlwaysOff)), _) => { + Some(opentelemetry::sdk::trace::Sampler::AlwaysOff) + } + (Some(SamplerOption::TraceIdRatioBased(ratio)), _) => Some( + opentelemetry::sdk::trace::Sampler::TraceIdRatioBased(*ratio), + ), + (_, _) => None, + }; + if let Some(sampler) = sampler { + trace_config = trace_config.with_sampler(sampler); + } + if let Some(n) = config.max_events_per_span { + trace_config = trace_config.with_max_events_per_span(n); + } + if let Some(n) = config.max_attributes_per_span { + trace_config = trace_config.with_max_attributes_per_span(n); + } + if let Some(n) = config.max_links_per_span { + trace_config = trace_config.with_max_links_per_span(n); + } + if let Some(n) = config.max_attributes_per_event { + trace_config = trace_config.with_max_attributes_per_event(n); + } + if let Some(n) = config.max_attributes_per_link { + trace_config = trace_config.with_max_attributes_per_link(n); + } + + let mut resource_defaults = vec![]; + if let Some(service_name) = &config.service_name { + resource_defaults.push(KeyValue::new( + opentelemetry_semantic_conventions::resource::SERVICE_NAME, + service_name.clone(), + )); + } + if let Some(service_namespace) = &config.service_namespace { + resource_defaults.push(KeyValue::new( + opentelemetry_semantic_conventions::resource::SERVICE_NAMESPACE, + service_namespace.clone(), + )); + } + let resource = Resource::new(resource_defaults).merge(&mut Resource::new( + config + .attributes + .clone() + .unwrap_or_default() + .iter() + .map(|(k, v)| { + KeyValue::new( + opentelemetry::Key::from(k.clone()), + opentelemetry::Value::from(v.clone()), + ) + }) + .collect::>(), + )); + + trace_config = trace_config.with_resource(resource); + trace_config + } +} + +fn parent_based(sampler: opentelemetry::sdk::trace::Sampler) -> opentelemetry::sdk::trace::Sampler { + opentelemetry::sdk::trace::Sampler::ParentBased(Box::new(sampler)) +} diff --git a/apollo-router/src/plugins/telemetry/metrics.rs b/apollo-router/src/plugins/telemetry/metrics.rs deleted file mode 100644 index 2c71d564a0..0000000000 --- a/apollo-router/src/plugins/telemetry/metrics.rs +++ /dev/null @@ -1,307 +0,0 @@ -use apollo_router_core::{ - http_compat, Handler, Plugin, ResponseBody, RouterRequest, RouterResponse, SubgraphRequest, - SubgraphResponse, -}; -use bytes::Bytes; -use futures::future::BoxFuture; -use http::{Method, StatusCode}; -use opentelemetry::{ - global, - metrics::{Counter, ValueRecorder}, - sdk::metrics::PushController, - KeyValue, -}; -use opentelemetry_prometheus::PrometheusExporter; -use prometheus::{Encoder, Registry, TextEncoder}; -use reqwest::Url; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use serde_json_bytes::from_value; -use std::{ - task::{Context, Poll}, - time::SystemTime, -}; -use tower::{service_fn, steer::Steer, util::BoxService, BoxError, Service, ServiceExt}; - -#[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] -use super::otlp::Metrics as OltpConfiguration; - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -pub struct MetricsConfiguration { - pub exporter: MetricsExporter, -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields, rename_all = "snake_case")] -pub enum MetricsExporter { - Prometheus(PrometheusConfiguration), - #[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] - Otlp(Box), -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -pub struct PrometheusConfiguration { - endpoint: String, -} - -#[derive(Debug)] -pub struct MetricsPlugin { - exporter: Option, - conf: MetricsConfiguration, - metrics_controller: Option, - router_metrics: BasicMetrics, - subgraph_metrics: BasicMetrics, -} - -#[derive(Debug)] -pub struct BasicMetrics { - http_requests_total: Counter, - http_requests_error_total: Counter, - http_requests_duration: ValueRecorder, -} - -#[async_trait::async_trait] -impl Plugin for MetricsPlugin { - type Config = MetricsConfiguration; - - fn new(config: Self::Config) -> Result { - let exporter = opentelemetry_prometheus::exporter().init(); - let meter = global::meter("apollo/router"); - - if let MetricsExporter::Prometheus(prom_exporter_cfg) = &config.exporter { - if Url::parse(&format!("http://test:8080{}", prom_exporter_cfg.endpoint)).is_err() { - return Err(BoxError::from( - "cannot use your endpoint set for prometheus as a path in an URL, your path need to be absolute (starting with a '/')", - )); - } - } - - Ok(Self { - exporter: exporter.into(), - conf: config, - router_metrics: BasicMetrics { - http_requests_total: meter - .u64_counter("http_requests_total") - .with_description("Total number of HTTP requests made.") - .init(), - http_requests_error_total: meter - .u64_counter("http_requests_error_total") - .with_description("Total number of HTTP requests in error made.") - .init(), - http_requests_duration: meter - .f64_value_recorder("http_request_duration_seconds") - .with_description("The HTTP request latencies in seconds.") - .init(), - }, - subgraph_metrics: BasicMetrics { - http_requests_total: meter - .u64_counter("http_requests_total_subgraph") - .with_description("Total number of HTTP requests made for a subgraph.") - .init(), - http_requests_error_total: meter - .u64_counter("http_requests_error_total_subgraph") - .with_description("Total number of HTTP requests in error made for a subgraph.") - .init(), - http_requests_duration: meter - .f64_value_recorder("http_request_duration_seconds_subgraph") - .with_description("The HTTP request latencies in seconds for a subgraph.") - .init(), - }, - metrics_controller: None, - }) - } - - async fn startup(&mut self) -> Result<(), BoxError> { - match &self.conf.exporter { - MetricsExporter::Prometheus(_) => {} - #[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] - MetricsExporter::Otlp(otlp_conf) => { - self.metrics_controller = otlp_conf.exporter.metrics_exporter()?.into(); - } - } - - Ok(()) - } - - fn router_service( - &mut self, - service: BoxService, - ) -> BoxService { - const METRICS_REQUEST_TIME: &str = "METRICS_REQUEST_TIME"; - let http_counter = self.router_metrics.http_requests_total.clone(); - let http_request_duration = self.router_metrics.http_requests_duration.clone(); - let http_requests_error_total = self.router_metrics.http_requests_error_total.clone(); - - service - .map_request(|req: RouterRequest| { - let request_start = SystemTime::now(); - req.context - .insert(METRICS_REQUEST_TIME, request_start) - .unwrap(); - - req - }) - .map_response(move |res| { - let request_start: SystemTime = from_value( - res.context - .extensions - .get(METRICS_REQUEST_TIME) - .unwrap() - .clone(), - ) - .unwrap(); - let kvs = &[ - KeyValue::new("url", res.context.request.url().to_string()), - KeyValue::new("status", res.response.status().as_u16().to_string()), - ]; - http_request_duration.record( - request_start.elapsed().map_or(0.0, |d| d.as_secs_f64()), - kvs, - ); - http_counter.add(1, kvs); - res - }) - .map_err(move |err: BoxError| { - http_requests_error_total.add(1, &[]); - - err - }) - .boxed() - } - - fn subgraph_service( - &mut self, - name: &str, - service: BoxService, - ) -> BoxService { - const METRICS_REQUEST_TIME: &str = "METRICS_REQUEST_TIME_SUBGRAPH"; - let subgraph_name = name.to_owned(); - let subgraph_name_cloned = name.to_owned(); - let subgraph_name_cloned_for_err = name.to_owned(); - let http_counter = self.subgraph_metrics.http_requests_total.clone(); - let http_request_duration = self.subgraph_metrics.http_requests_duration.clone(); - let http_requests_error_total = self.subgraph_metrics.http_requests_error_total.clone(); - let extension_metric_name = format!("{}_{}", METRICS_REQUEST_TIME, subgraph_name); - let extension_metric_name_cloned = extension_metric_name.clone(); - - service - .map_request(move |req: SubgraphRequest| { - let request_start = SystemTime::now(); - req.context - .insert(extension_metric_name.clone(), request_start) - .unwrap(); - - req - }) - .map_response(move |res| { - let request_start: SystemTime = from_value( - res.context - .extensions - .get(&extension_metric_name_cloned) - .unwrap() - .clone(), - ) - .unwrap(); - let kvs = &[ - KeyValue::new("subgraph", subgraph_name_cloned), - KeyValue::new("url", res.context.request.url().to_string()), - KeyValue::new("status", res.response.status().as_u16().to_string()), - ]; - http_request_duration.record( - request_start.elapsed().map_or(0.0, |d| d.as_secs_f64()), - kvs, - ); - http_counter.add(1, kvs); - res - }) - .map_err(move |err: BoxError| { - http_requests_error_total.add( - 1, - &[KeyValue::new("subgraph", subgraph_name_cloned_for_err)], - ); - - err - }) - .boxed() - } - - fn custom_endpoint(&self) -> Option { - let prometheus_endpoint = match &self.conf.exporter { - MetricsExporter::Prometheus(prom) => Some(prom.endpoint.clone()), - #[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] - MetricsExporter::Otlp(_) => None, - }; - - match (prometheus_endpoint, &self.exporter) { - (Some(endpoint), Some(exporter)) => { - let registry = exporter.registry().clone(); - - let not_found_handler = service_fn(|_req: http_compat::Request| async { - Ok::<_, BoxError>(http_compat::Response { - inner: http::Response::builder() - .status(StatusCode::NOT_FOUND) - .body(ResponseBody::Text(String::new())) - .unwrap(), - }) - }) - .boxed(); - let metrics_handler = PrometheusService { registry }.boxed(); - - let svc = Steer::new( - // All services we route between - vec![metrics_handler, not_found_handler], - // How we pick which service to send the request to - move |req: &http_compat::Request, _services: &[_]| { - if req.method() == Method::GET - && req - .url() - .path() - .trim_start_matches("/plugins/apollo.telemetry") - == endpoint - { - 0 // Index of `metrics handler` - } else { - 1 // Index of `not_found` - } - }, - ); - - Some(svc.boxed().into()) - } - _ => None, - } - } -} - -#[derive(Clone)] -pub struct PrometheusService { - registry: Registry, -} - -impl Service> for PrometheusService { - type Response = http_compat::Response; - type Error = BoxError; - type Future = BoxFuture<'static, Result>; - - fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { - Ok(()).into() - } - - fn call(&mut self, _req: http_compat::Request) -> Self::Future { - let encoder = TextEncoder::new(); - let metric_families = self.registry.gather(); - let mut result = Vec::new(); - encoder.encode(&metric_families, &mut result).unwrap(); - - Box::pin(async move { - Ok(http_compat::Response { - inner: http::Response::builder() - .status(StatusCode::OK) - .body(ResponseBody::Text( - String::from_utf8_lossy(&result).into_owned(), - )) - .map_err(|err| BoxError::from(err.to_string()))?, - }) - }) - } -} diff --git a/apollo-router/src/plugins/telemetry/metrics/mod.rs b/apollo-router/src/plugins/telemetry/metrics/mod.rs new file mode 100644 index 0000000000..0bb3165b01 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/metrics/mod.rs @@ -0,0 +1,162 @@ +use crate::plugins::telemetry::config::MetricsCommon; +use apollo_router_core::{http_compat, Handler, ResponseBody}; +use bytes::Bytes; +use opentelemetry::metrics::{Counter, Meter, MeterProvider, Number, ValueRecorder}; +use opentelemetry::KeyValue; +use std::any::Any; +use std::collections::HashMap; +use std::sync::Arc; +use tower::util::BoxService; +use tower::BoxError; + +pub mod otlp; +pub mod prometheus; + +pub type MetricsExporterHandle = Box; +pub type CustomEndpoint = + BoxService, http_compat::Response, BoxError>; + +#[derive(Default)] +pub struct MetricsBuilder { + exporters: Vec, + meter_providers: Vec>, + custom_endpoints: HashMap, +} + +impl MetricsBuilder { + pub fn exporters(&mut self) -> Vec { + std::mem::take(&mut self.exporters) + } + pub fn meter_provider(&mut self) -> AggregateMeterProvider { + AggregateMeterProvider::new(std::mem::take(&mut self.meter_providers)) + } + pub fn custom_endpoints(&mut self) -> HashMap { + std::mem::take(&mut self.custom_endpoints) + } +} + +impl MetricsBuilder { + fn with_exporter(mut self, handle: T) -> Self { + self.exporters.push(Box::new(handle)); + self + } + + fn with_meter_provider( + mut self, + meter_provider: T, + ) -> Self { + self.meter_providers.push(Arc::new(meter_provider)); + self + } + + fn with_custom_endpoint(mut self, path: &str, endpoint: CustomEndpoint) -> Self { + self.custom_endpoints + .insert(path.to_string(), Handler::new(endpoint)); + self + } +} + +pub trait MetricsConfigurator { + fn apply( + &self, + builder: MetricsBuilder, + metrics_config: &MetricsCommon, + ) -> Result; +} + +#[derive(Clone)] +pub struct BasicMetrics { + pub http_requests_total: AggregateCounter, + pub http_requests_error_total: AggregateCounter, + pub http_requests_duration: AggregateValueRecorder, +} + +impl BasicMetrics { + pub fn new(meter_provider: &AggregateMeterProvider) -> BasicMetrics { + let meter = meter_provider.meter("apollo/router", None); + BasicMetrics { + http_requests_total: meter.build_counter(|m| { + m.u64_counter("http_requests_total") + .with_description("Total number of HTTP requests made.") + .init() + }), + http_requests_error_total: meter.build_counter(|m| { + m.u64_counter("http_requests_error_total") + .with_description("Total number of HTTP requests in error made.") + .init() + }), + http_requests_duration: meter.build_value_recorder(|m| { + m.f64_value_recorder("http_request_duration_seconds") + .with_description("Total number of HTTP requests made.") + .init() + }), + } + } +} + +#[derive(Clone, Default)] +pub struct AggregateMeterProvider(Vec>); +impl AggregateMeterProvider { + pub fn new( + meters: Vec>, + ) -> AggregateMeterProvider { + AggregateMeterProvider(meters) + } + + pub fn meter( + &self, + instrumentation_name: &'static str, + instrumentation_version: Option<&'static str>, + ) -> AggregateMeter { + AggregateMeter( + self.0 + .iter() + .map(|p| Arc::new(p.meter(instrumentation_name, instrumentation_version))) + .collect(), + ) + } +} + +#[derive(Clone)] +pub struct AggregateMeter(Vec>); +impl AggregateMeter { + pub fn build_counter + Copy>( + &self, + build: fn(&Meter) -> Counter, + ) -> AggregateCounter { + AggregateCounter(self.0.iter().map(|m| build(m)).collect()) + } + + pub fn build_value_recorder + Copy>( + &self, + build: fn(&Meter) -> ValueRecorder, + ) -> AggregateValueRecorder { + AggregateValueRecorder(self.0.iter().map(|m| build(m)).collect()) + } +} + +#[derive(Clone)] +pub struct AggregateCounter + Copy>(Vec>); +impl AggregateCounter +where + T: Into + Copy, +{ + pub fn add(&self, value: T, attributes: &[KeyValue]) { + for counter in &self.0 { + counter.add(value, attributes) + } + } +} + +#[derive(Clone)] +pub struct AggregateValueRecorder + Copy>(Vec>); +impl AggregateValueRecorder +where + T: Into + Copy, +{ + pub fn record(&self, value: T, attributes: &[KeyValue]) { + for value_recorder in &self.0 { + value_recorder.record(value, attributes) + } + } +} diff --git a/apollo-router/src/plugins/telemetry/metrics/otlp.rs b/apollo-router/src/plugins/telemetry/metrics/otlp.rs new file mode 100644 index 0000000000..e153899255 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/metrics/otlp.rs @@ -0,0 +1,56 @@ +use crate::plugins::telemetry::config::MetricsCommon; +use crate::plugins::telemetry::metrics::{MetricsBuilder, MetricsConfigurator}; +use futures::{Stream, StreamExt}; +use opentelemetry::sdk::metrics::selectors; +use opentelemetry::util::tokio_interval_stream; +use opentelemetry_otlp::{HttpExporterBuilder, TonicExporterBuilder}; +use std::time::Duration; +use tower::BoxError; + +// TODO Remove MetricExporterBuilder once upstream issue is fixed +// This has to exist because Http is not currently supported for metrics export +// https://github.com/open-telemetry/opentelemetry-rust/issues/772 +struct MetricExporterBuilder { + exporter: Option, +} + +impl From for MetricExporterBuilder { + fn from(exporter: TonicExporterBuilder) -> Self { + Self { + exporter: Some(exporter), + } + } +} + +impl From for MetricExporterBuilder { + fn from(_exporter: HttpExporterBuilder) -> Self { + Self { exporter: None } + } +} + +impl MetricsConfigurator for super::super::otlp::Config { + fn apply( + &self, + mut builder: MetricsBuilder, + _metrics_config: &MetricsCommon, + ) -> Result { + let exporter: MetricExporterBuilder = self.exporter()?; + match exporter.exporter { + Some(exporter) => { + let exporter = opentelemetry_otlp::new_pipeline() + .metrics(tokio::spawn, delayed_interval) + .with_exporter(exporter) + .with_aggregator_selector(selectors::simple::Selector::Exact) + .build()?; + builder = builder.with_meter_provider(exporter.provider()); + builder = builder.with_exporter(exporter); + Ok(builder) + } + None => Err("otlp metric export does not support http yet".into()), + } + } +} + +fn delayed_interval(duration: Duration) -> impl Stream { + tokio_interval_stream(duration).skip(1) +} diff --git a/apollo-router/src/plugins/telemetry/metrics/prometheus.rs b/apollo-router/src/plugins/telemetry/metrics/prometheus.rs new file mode 100644 index 0000000000..51512ab3e2 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/metrics/prometheus.rs @@ -0,0 +1,72 @@ +use crate::future::BoxFuture; +use crate::plugins::telemetry::config::MetricsCommon; +use crate::plugins::telemetry::metrics::{MetricsBuilder, MetricsConfigurator}; +use apollo_router_core::{http_compat, ResponseBody}; +use bytes::Bytes; +use http::StatusCode; +use prometheus::{Encoder, Registry, TextEncoder}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::task::{Context, Poll}; +use tower::{BoxError, ServiceExt}; +use tower_service::Service; + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Config { + enabled: bool, +} + +impl MetricsConfigurator for Config { + fn apply( + &self, + mut builder: MetricsBuilder, + _metrics_config: &MetricsCommon, + ) -> Result { + if self.enabled { + let exporter = opentelemetry_prometheus::exporter().try_init()?; + builder = builder.with_custom_endpoint( + "/prometheus", + PrometheusService { + registry: exporter.registry().clone(), + } + .boxed(), + ); + builder = builder.with_meter_provider(exporter.provider()?); + builder = builder.with_exporter(exporter); + } + Ok(builder) + } +} + +#[derive(Clone)] +pub struct PrometheusService { + registry: Registry, +} + +impl Service> for PrometheusService { + type Response = http_compat::Response; + type Error = BoxError; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { + Ok(()).into() + } + + fn call(&mut self, _req: http_compat::Request) -> Self::Future { + let metric_families = self.registry.gather(); + Box::pin(async move { + let encoder = TextEncoder::new(); + let mut result = Vec::new(); + encoder.encode(&metric_families, &mut result)?; + Ok(http_compat::Response { + inner: http::Response::builder() + .status(StatusCode::OK) + .body(ResponseBody::Text( + String::from_utf8_lossy(&result).into_owned(), + )) + .map_err(|err| BoxError::from(err.to_string()))?, + }) + }) + } +} diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs index 8830be675f..c79cd3dd7d 100644 --- a/apollo-router/src/plugins/telemetry/mod.rs +++ b/apollo-router/src/plugins/telemetry/mod.rs @@ -1,220 +1,51 @@ //! Telemetry customization. - -pub(crate) mod metrics; -#[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] -pub(crate) mod otlp; - -use crate::apollo_telemetry::SpaceportConfig; -use crate::apollo_telemetry::StudioGraph; -use crate::apollo_telemetry::{new_pipeline, PipelineBuilder}; -use crate::configuration::{default_service_name, default_service_namespace}; -use crate::layers::opentracing::OpenTracingConfig; -use crate::layers::opentracing::OpenTracingLayer; -use crate::subscriber::{replace_layer, BaseLayer, BoxedLayer}; -use apollo_router_core::Handler; -use apollo_router_core::RouterRequest; -use apollo_router_core::RouterResponse; -use apollo_router_core::SubgraphRequest; -use apollo_router_core::SubgraphResponse; -use apollo_router_core::{register_plugin, Plugin}; +use crate::plugins::telemetry::config::{MetricsCommon, Trace}; +use crate::plugins::telemetry::metrics::{ + AggregateMeterProvider, BasicMetrics, MetricsBuilder, MetricsConfigurator, + MetricsExporterHandle, +}; +use crate::plugins::telemetry::tracing::{apollo, TracingConfigurator}; +use crate::subscriber::replace_layer; +use apollo_router_core::{ + http_compat, register_plugin, Handler, Plugin, ResponseBody, RouterRequest, RouterResponse, + SubgraphRequest, SubgraphResponse, +}; use apollo_spaceport::server::ReportSpaceport; -use derivative::Derivative; -use futures::Future; -use opentelemetry::sdk::trace::{BatchSpanProcessor, Sampler}; -use opentelemetry::sdk::Resource; -use opentelemetry::trace::TracerProvider; -use opentelemetry::{Array, KeyValue, Value}; -#[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] -use otlp::Tracing; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::borrow::Cow; -use std::collections::BTreeMap; +use bytes::Bytes; +use futures::FutureExt; +use http::StatusCode; +use opentelemetry::propagation::TextMapPropagator; +use opentelemetry::sdk::propagation::{ + BaggagePropagator, TextMapCompositePropagator, TraceContextPropagator, +}; +use opentelemetry::sdk::trace::Builder; +use opentelemetry::trace::{Tracer, TracerProvider}; +use opentelemetry::{global, KeyValue}; +use std::collections::HashMap; use std::error::Error; use std::fmt; -use std::net::SocketAddr; -use std::pin::Pin; -use std::str::FromStr; +use std::time::Instant; +use tower::steer::Steer; use tower::util::BoxService; -use tower::Layer; -use tower::{BoxError, ServiceExt}; +use tower::{service_fn, BoxError, ServiceExt}; use url::Url; -use self::metrics::MetricsConfiguration; -#[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] -use self::metrics::MetricsExporter; -use self::metrics::MetricsPlugin; - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields, rename_all = "snake_case")] -#[allow(clippy::large_enum_variant)] -pub enum OpenTelemetry { - Jaeger(Option), - #[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] - Otlp(otlp::Otlp), -} - -#[derive(Debug, Clone, Derivative, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields)] -#[derivative(Default)] -pub struct Jaeger { - pub endpoint: Option, - #[serde(default = "default_service_name")] - #[derivative(Default(value = "default_service_name()"))] - pub service_name: String, - #[serde(skip, default = "default_jaeger_username")] - #[derivative(Default(value = "default_jaeger_username()"))] - pub username: Option, - #[serde(skip, default = "default_jaeger_password")] - #[derivative(Default(value = "default_jaeger_password()"))] - pub password: Option, - pub trace_config: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields, rename_all = "snake_case")] -pub enum JaegerEndpoint { - Agent(SocketAddr), - Collector(Url), -} - -fn default_jaeger_username() -> Option { - std::env::var("JAEGER_USERNAME").ok() -} - -fn default_jaeger_password() -> Option { - std::env::var("JAEGER_PASSWORD").ok() -} - -#[derive(Debug, Default, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct TraceConfig { - #[schemars(schema_with = "option_sampler_schema", default)] - pub sampler: Option, - pub max_events_per_span: Option, - pub max_attributes_per_span: Option, - pub max_links_per_span: Option, - pub max_attributes_per_event: Option, - pub max_attributes_per_link: Option, - pub attributes: Option>, -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(untagged, deny_unknown_fields)] -pub enum AttributeValue { - /// bool values - Bool(bool), - /// i64 values - I64(i64), - /// f64 values - F64(f64), - /// String values - String(String), - /// Array of homogeneous values - Array(AttributeArray), -} - -impl From for opentelemetry::Value { - fn from(value: AttributeValue) -> Self { - match value { - AttributeValue::Bool(v) => Value::Bool(v), - AttributeValue::I64(v) => Value::I64(v), - AttributeValue::F64(v) => Value::F64(v), - AttributeValue::String(v) => Value::String(Cow::from(v)), - AttributeValue::Array(v) => Value::Array(v.into()), - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(untagged, deny_unknown_fields)] -pub enum AttributeArray { - /// Array of bools - Bool(Vec), - /// Array of integers - I64(Vec), - /// Array of floats - F64(Vec), - /// Array of strings - String(Vec>), -} - -impl From for opentelemetry::Array { - fn from(array: AttributeArray) -> Self { - match array { - AttributeArray::Bool(v) => Array::Bool(v), - AttributeArray::I64(v) => Array::I64(v), - AttributeArray::F64(v) => Array::F64(v), - AttributeArray::String(v) => Array::String(v), - } - } -} - -fn option_sampler_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema { - Option::::json_schema(gen) -} - -#[derive(JsonSchema)] -#[allow(dead_code)] -pub enum SamplerMirror { - /// Always sample the trace - AlwaysOn, - /// Never sample the trace - AlwaysOff, - /// Respects the parent span's sampling decision or delegates a delegate sampler for root spans. - /// Not supported via yaml config - //ParentBased(Box), - /// Sample a given fraction of traces. Fractions >= 1 will always sample. If the parent span is - /// sampled, then it's child spans will automatically be sampled. Fractions < 0 are treated as - /// zero, but spans may still be sampled if their parent is. - TraceIdRatioBased(f64), -} - -impl TraceConfig { - pub fn trace_config(&self) -> opentelemetry::sdk::trace::Config { - let mut trace_config = opentelemetry::sdk::trace::config(); - if let Some(sampler) = self.sampler.clone() { - let sampler: opentelemetry::sdk::trace::Sampler = sampler; - trace_config = trace_config.with_sampler(sampler); - } - if let Some(n) = self.max_events_per_span { - trace_config = trace_config.with_max_events_per_span(n); - } - if let Some(n) = self.max_attributes_per_span { - trace_config = trace_config.with_max_attributes_per_span(n); - } - if let Some(n) = self.max_links_per_span { - trace_config = trace_config.with_max_links_per_span(n); - } - if let Some(n) = self.max_attributes_per_event { - trace_config = trace_config.with_max_attributes_per_event(n); - } - if let Some(n) = self.max_attributes_per_link { - trace_config = trace_config.with_max_attributes_per_link(n); - } - - let resource = Resource::new(vec![ - KeyValue::new("service.name", default_service_name()), - KeyValue::new("service.namespace", default_service_namespace()), - ]) - .merge(&mut Resource::new( - self.attributes - .clone() - .unwrap_or_default() - .iter() - .map(|(k, v)| { - KeyValue::new( - opentelemetry::Key::from(k.clone()), - opentelemetry::Value::from(v.clone()), - ) - }) - .collect::>(), - )); - - trace_config = trace_config.with_resource(resource); - - trace_config - } +mod config; +mod metrics; +mod otlp; +mod tracing; + +pub struct Telemetry { + config: config::Conf, + tracer_provider: Option, + + // Do not remove _metrics_exporters. Metrics will not be exported if it is removed. + // Typically the handles are a PushController but may be something else. Dropping the handle will + // shutdown exporter. + _metrics_exporters: Vec, + meter_provider: AggregateMeterProvider, + custom_endpoints: HashMap, + spaceport_shutdown: Option>, } #[derive(Debug)] @@ -228,139 +59,148 @@ impl fmt::Display for ReportingError { impl std::error::Error for ReportingError {} -#[derive(Debug)] -struct Telemetry { - config: Conf, - tx: tokio::sync::mpsc::Sender, - opentracing_layer: Option, - metrics_plugin: Option, +fn setup_tracing( + mut builder: Builder, + configurator: &Option, + tracing_config: &Trace, +) -> Result { + if let Some(config) = configurator { + builder = config.apply(builder, tracing_config)?; + } + Ok(builder) } -#[derive(Debug, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields, rename_all = "snake_case")] -struct Conf { - pub spaceport: Option, - - #[serde(skip, default)] - pub graph: Option, - - pub opentelemetry: Option, - - pub opentracing: Option, - - pub metrics: Option, +fn setup_metrics_exporter( + mut builder: MetricsBuilder, + configurator: &Option, + metrics_common: &MetricsCommon, +) -> Result { + if let Some(config) = configurator { + builder = config.apply(builder, metrics_common)?; + } + Ok(builder) } -fn studio_graph() -> Option { - if let Ok(apollo_key) = std::env::var("APOLLO_KEY") { - let apollo_graph_ref = std::env::var("APOLLO_GRAPH_REF").expect( - "cannot set up usage reporting if the APOLLO_GRAPH_REF environment variable is not set", - ); +fn apollo_key() -> Option { + std::env::var("APOLLO_KEY").ok() +} - Some(StudioGraph { - reference: apollo_graph_ref, - key: apollo_key, - }) - } else { - None - } +fn apollo_graph_reference() -> Option { + std::env::var("APOLLO_GRAPH_REF").ok() } #[async_trait::async_trait] impl Plugin for Telemetry { - type Config = Conf; + type Config = config::Conf; async fn startup(&mut self) -> Result<(), BoxError> { - replace_layer(self.try_build_layer()?)?; - - // Only check for notify if we have graph configuration - if self.config.graph.is_some() { - self.tx - .send(self.config.spaceport.clone().unwrap_or_default()) - .await?; + // Apollo config is special because we enable tracing if some env variables are present. + let apollo = self.config.apollo.get_or_insert_with(Default::default); + if apollo.apollo_key.is_none() { + apollo.apollo_key = apollo_key() + } + if apollo.apollo_graph_ref.is_none() { + apollo.apollo_graph_ref = apollo_graph_reference() } - #[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] - if let Some(MetricsExporter::Otlp(_otlp_exporter_conf)) = &mut self - .config - .metrics - .as_mut() - .map(|m_conf| &mut m_conf.exporter) - { - self.metrics_plugin - .as_mut() - .expect("configuration has already been checked in the new method; qed") - .startup() + // If we have key and graph ref but no endpoint we start embedded spaceport + let (spaceport, shutdown_tx) = match apollo { + apollo::Config { + apollo_key: Some(_), + apollo_graph_ref: Some(_), + endpoint: None, + } => { + ::tracing::debug!("starting Spaceport"); + let (shutdown_tx, shutdown_rx) = futures::channel::oneshot::channel(); + let report_spaceport = ReportSpaceport::new( + "127.0.0.1:0".parse()?, + Some(Box::pin(shutdown_rx.map(|_| ()))), + ) .await?; + // Now that the port is known update the config + apollo.endpoint = Some(Url::parse(&format!( + "https://{}", + report_spaceport.address() + ))?); + (Some(report_spaceport), Some(shutdown_tx)) + } + _ => (None, None), + }; + + //If store the shutdown handle. + self.spaceport_shutdown = shutdown_tx; + + // Now that spaceport is set up it is possible to set up the tracer providers. + self.tracer_provider = Some(Self::create_tracer_provider(&self.config)?); + + // Setup metrics + // The act of setting up metrics will overwrite a global meter. However it is essential that + // we use the aggregate meter provider that is created below. It enables us to support + // sending metrics to multiple providers at once, of which hopefully Apollo Studio will + // eventually be one. + let mut builder = Self::create_metrics_exporters(&self.config)?; + self._metrics_exporters = builder.exporters(); + self.meter_provider = builder.meter_provider(); + self.custom_endpoints = builder.custom_endpoints(); + + // Finally actually start spaceport + if let Some(spaceport) = spaceport { + tokio::spawn(async move { + if let Err(e) = spaceport.serve().await { + match e.source() { + Some(source) => { + ::tracing::warn!("spaceport did not terminate normally: {}", source); + } + None => { + ::tracing::warn!("spaceport did not terminate normally: {}", e); + } + } + }; + }); } - Ok(()) - } - async fn shutdown(&mut self) -> Result<(), BoxError> { Ok(()) } - fn new(mut configuration: Self::Config) -> Result { - // Graph can only be set via env variables. - configuration.graph = studio_graph(); - tracing::debug!("Apollo graph configuration: {:?}", configuration.graph); - // Studio Agent Spaceport listener - let (tx, mut rx) = tokio::sync::mpsc::channel::(1); - - tokio::spawn(async move { - let mut current_listener = "".to_string(); - let mut current_operation: fn( - msg: String, - ) - -> Pin + Send>> = |msg| Box::pin(do_nothing(msg)); - - loop { - tokio::select! { - biased; - mopt = rx.recv() => { - match mopt { - Some(msg) => { - tracing::debug!(?msg); - // Save our target listener for later use - current_listener = msg.listener.clone(); - // Configure which function to call - if msg.external { - current_operation = |msg| Box::pin(do_nothing(msg)); - } else { - current_operation = |msg| Box::pin(do_listen(msg)); - } - }, - None => break - } - }, - x = current_operation(current_listener.clone()) => { - // current_operation will only return if there is - // something wrong in our configuration. We don't - // want to terminate, so wait for a while and - // then try again. At some point, re-configuration - // will fix this. - tracing::debug!(%x, "current_operation"); - tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - } - }; - } - tracing::debug!("terminating spaceport loop"); - }); + fn activate(&mut self) { + // The active service is about to be swapped in. + // The rest of this code in this method is expected to succeed. + // The issue is that Otel uses globals for a bunch of stuff. + // If we move to a completely tower based architecture then we could make this nicer. + let tracer_provider = self + .tracer_provider + .take() + .expect("trace_provider will have been set in startup, qed"); + let tracer = tracer_provider.versioned_tracer( + "apollo-router", + Some(env!("CARGO_PKG_VERSION")), + None, + ); + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + Self::replace_tracer_provider(tracer_provider); - let mut opentracing_layer = None; - if let Some(opentracing_conf) = &configuration.opentracing { - opentracing_layer = OpenTracingLayer::new(opentracing_conf.clone()).into(); - } - let mut metrics_plugin = None; - if let Some(metrics_conf) = &configuration.metrics { - metrics_plugin = MetricsPlugin::new(metrics_conf.clone())?.into(); + replace_layer(Box::new(telemetry)) + .expect("set_global_subscriber() was not called at startup, fatal"); + opentelemetry::global::set_error_handler(handle_error) + .expect("otel error handler lock poisoned, fatal"); + global::set_text_map_propagator(Self::create_propagator(&self.config)); + } + + async fn shutdown(&mut self) -> Result<(), BoxError> { + if let Some(sender) = self.spaceport_shutdown.take() { + let _ = sender.send(()); } + Ok(()) + } + fn new(config: Self::Config) -> Result { Ok(Telemetry { - config: configuration, - tx, - opentracing_layer, - metrics_plugin, + spaceport_shutdown: None, + tracer_provider: None, + custom_endpoints: Default::default(), + _metrics_exporters: Default::default(), + meter_provider: Default::default(), + config, }) } @@ -368,321 +208,242 @@ impl Plugin for Telemetry { &mut self, service: BoxService, ) -> BoxService { - match &mut self.metrics_plugin { - Some(metrics_plugin) => metrics_plugin.router_service(service), - None => service, - } + let metrics = BasicMetrics::new(&self.meter_provider); + service + .map_future(move |f| { + let metrics = metrics.clone(); + // Using Instant because it is guaranteed to be monotonically increasing. + let now = Instant::now(); + f.map(move |r| { + match &r { + Ok(response) => { + metrics.http_requests_total.add( + 1, + &[KeyValue::new( + "status", + response.response.status().as_u16().to_string(), + )], + ); + } + Err(_) => { + metrics.http_requests_error_total.add(1, &[]); + } + } + metrics + .http_requests_duration + .record(now.elapsed().as_secs_f64(), &[]); + r + }) + }) + .boxed() } fn subgraph_service( &mut self, name: &str, - mut service: BoxService, + service: BoxService, ) -> BoxService { - service = match &self.opentracing_layer { - Some(opentracing_layer) => opentracing_layer.layer(service).boxed(), - None => service, - }; - - match &mut self.metrics_plugin { - Some(metrics_plugin) => metrics_plugin.subgraph_service(name, service), - None => service, - } + let metrics = BasicMetrics::new(&self.meter_provider); + let subgraph_attribute = KeyValue::new("subgraph", name.to_string()); + service + .map_future(move |f| { + let metrics = metrics.clone(); + let subgraph_attribute = subgraph_attribute.clone(); + // Using Instant because it is guaranteed to be monotonically increasing. + let now = Instant::now(); + f.map(move |r| { + match &r { + Ok(response) => { + metrics.http_requests_total.add( + 1, + &[ + KeyValue::new( + "status", + response.response.status().as_u16().to_string(), + ), + subgraph_attribute.clone(), + ], + ); + } + Err(_) => { + metrics + .http_requests_error_total + .add(1, &[subgraph_attribute.clone()]); + } + } + metrics + .http_requests_duration + .record(now.elapsed().as_secs_f64(), &[subgraph_attribute.clone()]); + r + }) + }) + .boxed() } fn custom_endpoint(&self) -> Option { - match &self.metrics_plugin { - Some(metrics_plugin) => metrics_plugin.custom_endpoint(), - None => None, - } + let (paths, mut endpoints): (Vec<_>, Vec<_>) = + self.custom_endpoints.clone().into_iter().unzip(); + endpoints.push(Self::not_found_endpoint()); + let not_found_index = endpoints.len() - 1; + + let svc = Steer::new( + // All services we route between + endpoints, + // How we pick which service to send the request to + move |req: &http_compat::Request, _services: &[_]| { + let endpoint = req + .url() + .path() + .trim_start_matches("/plugins/apollo.telemetry"); + if let Some(index) = paths.iter().position(|path| path == endpoint) { + index + } else { + not_found_index + } + }, + ) + .boxed(); + + Some(Handler::new(svc)) } } impl Telemetry { - fn try_build_layer(&self) -> Result { - let spaceport_config = &self.config.spaceport; - let graph_config = &self.config.graph; - - match self.config.opentelemetry.as_ref() { - Some(OpenTelemetry::Jaeger(config)) => { - Self::setup_jaeger(spaceport_config, graph_config, config) - } - #[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] - Some(OpenTelemetry::Otlp(otlp::Otlp { - tracing: Some(tracing), - })) => Self::setup_otlp(spaceport_config, graph_config, tracing), - _ => Self::setup_spaceport(spaceport_config, graph_config), + fn create_propagator(config: &config::Conf) -> TextMapCompositePropagator { + let propagation = config + .clone() + .tracing + .and_then(|c| c.propagation) + .unwrap_or_default(); + + let tracing = config.clone().tracing.unwrap_or_default(); + + let mut propagators: Vec> = Vec::new(); + if propagation.baggage.unwrap_or_default() { + propagators.push(Box::new(BaggagePropagator::default())); } - } - - fn setup_spaceport( - spaceport_config: &Option, - graph_config: &Option, - ) -> Result { - if graph_config.is_some() { - // Add spaceport agent as an OT pipeline - let apollo_exporter = - Self::apollo_exporter_pipeline(spaceport_config, graph_config).install_batch()?; - let agent = tracing_opentelemetry::layer().with_tracer(apollo_exporter); - tracing::debug!("adding agent telemetry"); - Ok(Box::new(agent)) - } else { - // If we don't have any reporting to do, just put in place our BaseLayer - // (which does nothing) - Ok(Box::new(BaseLayer {})) + if propagation.trace_context.unwrap_or_default() || tracing.otlp.is_some() { + propagators.push(Box::new(TraceContextPropagator::default())); } - } - - fn apollo_exporter_pipeline( - spaceport_config: &Option, - graph_config: &Option, - ) -> PipelineBuilder { - new_pipeline() - .with_spaceport_config(spaceport_config) - .with_graph_config(graph_config) - } - - #[cfg(any(feature = "otlp-grpc", feature = "otlp-http"))] - fn setup_otlp( - spaceport_config: &Option, - graph_config: &Option, - config: &Tracing, - ) -> Result { - let batch_size = std::env::var("OTEL_BSP_MAX_EXPORT_BATCH_SIZE") - .ok() - .and_then(|batch_size| usize::from_str(&batch_size).ok()); - - let batch = BatchSpanProcessor::builder( - config.exporter.exporter()?.build_span_exporter()?, - opentelemetry::runtime::Tokio, - ) - .with_scheduled_delay(std::time::Duration::from_secs(1)); - let batch = if let Some(size) = batch_size { - batch.with_max_export_batch_size(size) - } else { - batch + if propagation.zipkin.unwrap_or_default() || tracing.zipkin.is_some() { + propagators.push(Box::new(opentelemetry_zipkin::Propagator::default())); } - .build(); - - let mut builder = opentelemetry::sdk::trace::TracerProvider::builder(); - builder = builder.with_config( - config - .trace_config - .clone() - .unwrap_or_default() - .trace_config(), - ); - - // If we have apollo graph configuration, then we can export statistics - // to the apollo ingress. If we don't, we can't and so no point configuring the - // exporter. - if graph_config.is_some() { - let apollo_exporter = - Self::apollo_exporter_pipeline(spaceport_config, graph_config).get_exporter()?; - builder = builder.with_batch_exporter(apollo_exporter, opentelemetry::runtime::Tokio) + if propagation.jaeger.unwrap_or_default() || tracing.jaeger.is_some() { + propagators.push(Box::new(opentelemetry_jaeger::Propagator::default())); } - - let provider = builder.with_span_processor(batch).build(); - - let tracer = - provider.versioned_tracer("opentelemetry-otlp", Some(env!("CARGO_PKG_VERSION")), None); - - // This code will hang unless we execute from a separate - // thread. See: - // https://github.com/apollographql/router/issues/331 - // https://github.com/open-telemetry/opentelemetry-rust/issues/536 - // for more details and description. - let jh = tokio::task::spawn_blocking(|| { - opentelemetry::global::force_flush_tracer_provider(); - opentelemetry::global::set_tracer_provider(provider); - }); - futures::executor::block_on(jh)?; - - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - - opentelemetry::global::set_error_handler(handle_error)?; - - Ok(Box::new(telemetry)) - } - - fn setup_jaeger( - spaceport_config: &Option, - graph_config: &Option, - config: &Option, - ) -> Result { - let default_config = Default::default(); - let config = config.as_ref().unwrap_or(&default_config); - let mut pipeline = - opentelemetry_jaeger::new_pipeline().with_service_name(&config.service_name); - match config.endpoint.as_ref() { - Some(JaegerEndpoint::Agent(address)) => { - pipeline = pipeline.with_agent_endpoint(address) - } - Some(JaegerEndpoint::Collector(url)) => { - pipeline = pipeline.with_collector_endpoint(url.as_str()); - - if let Some(username) = config.username.as_ref() { - pipeline = pipeline.with_collector_username(username); - } - if let Some(password) = config.password.as_ref() { - pipeline = pipeline.with_collector_password(password); - } - } - _ => {} + if propagation.datadog.unwrap_or_default() || tracing.datadog.is_some() { + propagators.push(Box::new(opentelemetry_datadog::DatadogPropagator::default())); } - let batch_size = std::env::var("OTEL_BSP_MAX_EXPORT_BATCH_SIZE") - .ok() - .and_then(|batch_size| usize::from_str(&batch_size).ok()); - - let exporter = pipeline.init_async_exporter(opentelemetry::runtime::Tokio)?; - - let batch = BatchSpanProcessor::builder(exporter, opentelemetry::runtime::Tokio) - .with_scheduled_delay(std::time::Duration::from_secs(1)); - let batch = if let Some(size) = batch_size { - batch.with_max_export_batch_size(size) - } else { - batch - } - .build(); + TextMapCompositePropagator::new(propagators) + } - let mut builder = opentelemetry::sdk::trace::TracerProvider::builder(); - if let Some(trace_config) = &config.trace_config { - builder = builder.with_config(trace_config.trace_config()); - } - // If we have apollo graph configuration, then we can export statistics - // to the apollo ingress. If we don't, we can't and so no point configuring the - // exporter. - if graph_config.is_some() { - let apollo_exporter = - Self::apollo_exporter_pipeline(spaceport_config, graph_config).get_exporter()?; - builder = builder.with_batch_exporter(apollo_exporter, opentelemetry::runtime::Tokio) - } + fn create_tracer_provider( + config: &config::Conf, + ) -> Result { + let tracing_config = config.tracing.clone().unwrap_or_default(); + let trace_config = &tracing_config.trace_config.unwrap_or_default(); + let mut builder = + opentelemetry::sdk::trace::TracerProvider::builder().with_config(trace_config.into()); + + builder = setup_tracing(builder, &tracing_config.jaeger, trace_config)?; + builder = setup_tracing(builder, &tracing_config.zipkin, trace_config)?; + builder = setup_tracing(builder, &tracing_config.datadog, trace_config)?; + builder = setup_tracing(builder, &tracing_config.otlp, trace_config)?; + builder = setup_tracing(builder, &config.apollo, trace_config)?; + let tracer_provider = builder.build(); + Ok(tracer_provider) + } - let provider = builder.with_span_processor(batch).build(); + fn create_metrics_exporters(config: &config::Conf) -> Result { + let metrics_config = config.metrics.clone().unwrap_or_default(); + let metrics_common_config = &metrics_config.common.unwrap_or_default(); + let mut builder = MetricsBuilder::default(); + builder = + setup_metrics_exporter(builder, &metrics_config.prometheus, metrics_common_config)?; + builder = setup_metrics_exporter(builder, &metrics_config.otlp, metrics_common_config)?; + Ok(builder) + } - let tracer = provider.versioned_tracer( - "opentelemetry-jaeger", - Some(env!("CARGO_PKG_VERSION")), - None, - ); + fn not_found_endpoint() -> Handler { + Handler::new( + service_fn(|_req: http_compat::Request| async { + Ok::<_, BoxError>(http_compat::Response { + inner: http::Response::builder() + .status(StatusCode::NOT_FOUND) + .body(ResponseBody::Text("Not found".to_string())) + .unwrap(), + }) + }) + .boxed(), + ) + } - // This code will hang unless we execute from a separate - // thread. See: - // https://github.com/apollographql/router/issues/331 - // https://github.com/open-telemetry/opentelemetry-rust/issues/536 - // for more details and description. + fn replace_tracer_provider(tracer_provider: T) + where + T: TracerProvider + Send + Sync + 'static, + ::Tracer: Send + Sync + 'static, + <::Tracer as Tracer>::Span: + Send + Sync + 'static, + { let jh = tokio::task::spawn_blocking(|| { opentelemetry::global::force_flush_tracer_provider(); - opentelemetry::global::set_tracer_provider(provider); + opentelemetry::global::set_tracer_provider(tracer_provider); }); - futures::executor::block_on(jh)?; - - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - - opentelemetry::global::set_error_handler(handle_error)?; - - Ok(Box::new(telemetry)) + futures::executor::block_on(jh).expect("failed to replace tracer provider"); } } fn handle_error>(err: T) { match err.into() { opentelemetry::global::Error::Trace(err) => { - tracing::error!("OpenTelemetry trace error occurred: {}", err) + ::tracing::error!("OpenTelemetry trace error occurred: {}", err) } opentelemetry::global::Error::Other(err_msg) => { - tracing::error!("OpenTelemetry error occurred: {}", err_msg) + ::tracing::error!("OpenTelemetry error occurred: {}", err_msg) } other => { - tracing::error!("OpenTelemetry error occurred: {:?}", other) + ::tracing::error!("OpenTelemetry error occurred: {:?}", other) } } } -// For use when we have an external collector. Makes selecting over -// events simpler -async fn do_nothing(_addr_str: String) -> bool { - loop { - tokio::time::sleep(tokio::time::Duration::from_secs(3600)).await; - } - #[allow(unreachable_code)] - false -} - -// For use when we have an internal collector. -async fn do_listen(addr_str: String) -> bool { - tracing::debug!("spawning an internal spaceport"); - // Spawn a spaceport server to handle statistics - let addr = match addr_str.parse() { - Ok(a) => a, - Err(e) => { - tracing::warn!("could not parse spaceport address: {}", e); - return false; - } - }; - - let spaceport = ReportSpaceport::new(addr); - - if let Err(e) = spaceport.serve().await { - match e.source() { - Some(source) => { - tracing::warn!("spaceport did not terminate normally: {}", source); - } - None => { - tracing::warn!("spaceport did not terminate normally: {}", e); - } - } - return false; - } - true -} - register_plugin!("apollo", "telemetry", Telemetry); #[cfg(test)] mod tests { - #[tokio::test] async fn plugin_registered() { apollo_router_core::plugins() .get("apollo.telemetry") .expect("Plugin not found") - .create_instance(&serde_json::json!({ "opentelemetry": null })) + .create_instance(&serde_json::json!({ "tracing": null })) .unwrap(); } #[tokio::test] - #[cfg(any(feature = "otlp-grpc"))] async fn attribute_serialization() { apollo_router_core::plugins() .get("apollo.telemetry") .expect("Plugin not found") - .create_instance(&serde_json::json!({ "opentelemetry": { - "otlp": { - "tracing": { - "exporter": { - "grpc": { - "protocol": "Grpc" - }, - }, - "trace_config": { - "attributes": { - "str": "a", - "int": 1, - "float": 1.0, - "bool": true, - "str_arr": ["a", "b"], - "int_arr": [1, 2], - "float_arr": [1.0, 2.0], - "bool_arr": [true, false] - } + .create_instance(&serde_json::json!({ + "tracing": { + + "trace_config": { + "service_name": "router", + "attributes": { + "str": "a", + "int": 1, + "float": 1.0, + "bool": true, + "str_arr": ["a", "b"], + "int_arr": [1, 2], + "float_arr": [1.0, 2.0], + "bool_arr": [true, false] } } } - } - } - )) + })) .unwrap(); } } diff --git a/apollo-router/src/plugins/telemetry/otlp.rs b/apollo-router/src/plugins/telemetry/otlp.rs new file mode 100644 index 0000000000..ccf0790761 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/otlp.rs @@ -0,0 +1,237 @@ +use crate::configuration::ConfigurationError; +use crate::plugins::telemetry::config::GenericWith; +use opentelemetry_otlp::{HttpExporterBuilder, TonicExporterBuilder, WithExportConfig}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::Duration; +use tonic::metadata::MetadataMap; +use tonic::transport::ClientTlsConfig; +use tower::BoxError; +use url::Url; + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Config { + pub endpoint: Endpoint, + pub protocol: Option, + + #[serde(deserialize_with = "humantime_serde::deserialize", default)] + #[schemars(with = "String", default)] + pub timeout: Option, + pub grpc: Option, + pub http: Option, +} + +impl Config { + pub fn exporter + From>( + &self, + ) -> Result { + let endpoint = match &self.endpoint { + Endpoint::Default(_) => None, + Endpoint::Url(s) => Some(s), + }; + match self.protocol.clone().unwrap_or_default() { + Protocol::Grpc => { + let grpc = self.grpc.clone().unwrap_or_default(); + let exporter = opentelemetry_otlp::new_exporter() + .tonic() + .with_env() + .with(&self.timeout, |b, t| b.with_timeout(*t)) + .with(&endpoint, |b, e| b.with_endpoint(e.as_str())) + .try_with( + &grpc.tls_config, + |b, t| Ok(b.with_tls_config(t.try_into()?)), + )? + .with(&grpc.metadata, |b, m| b.with_metadata(m.clone())) + .into(); + Ok(exporter) + } + Protocol::Http => { + let http = self.http.clone().unwrap_or_default(); + let exporter = opentelemetry_otlp::new_exporter() + .http() + .with_env() + .with(&self.timeout, |b, t| b.with_timeout(*t)) + .with(&endpoint, |b, e| b.with_endpoint(e.as_str())) + .with(&http.headers, |b, h| b.with_headers(h.clone())) + .into(); + + Ok(exporter) + } + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case", untagged)] +pub enum Endpoint { + Default(EndpointDefault), + Url(Url), +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum EndpointDefault { + Default, +} + +#[derive(Debug, Clone, Deserialize, Serialize, Default, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct HttpExporter { + pub headers: Option>, +} + +#[derive(Debug, Clone, Deserialize, Serialize, Default, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct GrpcExporter { + #[serde(flatten)] + pub tls_config: Option, + #[serde( + deserialize_with = "metadata_map_serde::deserialize", + serialize_with = "metadata_map_serde::serialize", + default + )] + #[schemars(schema_with = "option_metadata_map", default)] + pub metadata: Option, +} + +fn option_metadata_map(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema { + Option::>::json_schema(gen) +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct TlsConfig { + domain_name: Option, + ca: Option, + cert: Option, + key: Option, +} + +impl TryFrom<&TlsConfig> for tonic::transport::channel::ClientTlsConfig { + type Error = BoxError; + + fn try_from(config: &TlsConfig) -> Result { + ClientTlsConfig::new() + .with(&config.domain_name, |b, d| b.domain_name(d)) + .try_with(&config.ca, |b, c| { + Ok(b.ca_certificate(tonic::transport::Certificate::from_pem(c.read()?))) + })? + .try_with( + &config.cert.clone().zip(config.key.clone()), + |b, (cert, key)| { + Ok(b.identity(tonic::transport::Identity::from_pem( + cert.read()?, + key.read()?, + ))) + }, + ) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum Secret { + Env(String), + File(PathBuf), +} + +impl Secret { + pub fn read(&self) -> Result { + match self { + Secret::Env(s) => std::env::var(s).map_err(ConfigurationError::CannotReadSecretFromEnv), + Secret::File(path) => { + std::fs::read_to_string(path).map_err(ConfigurationError::CannotReadSecretFromFile) + } + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum Protocol { + Grpc, + Http, +} + +impl Default for Protocol { + fn default() -> Self { + Protocol::Grpc + } +} + +mod metadata_map_serde { + use super::*; + use std::collections::HashMap; + use tonic::metadata::{KeyAndValueRef, MetadataKey}; + + pub(crate) fn serialize(map: &Option, serializer: S) -> Result + where + S: serde::ser::Serializer, + { + if map.as_ref().map(|x| x.is_empty()).unwrap_or(true) { + return serializer.serialize_none(); + } + + let mut serializable_format = + Vec::with_capacity(map.as_ref().map(|x| x.len()).unwrap_or(0)); + + serializable_format.extend(map.iter().flat_map(|x| x.iter()).map(|key_and_value| { + match key_and_value { + KeyAndValueRef::Ascii(key, value) => { + let mut map = HashMap::with_capacity(1); + map.insert(key.as_str(), value.to_str().unwrap()); + map + } + KeyAndValueRef::Binary(_, _) => todo!(), + } + })); + + serializable_format.serialize(serializer) + } + + pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: serde::de::Deserializer<'de>, + { + let serializable_format: Vec> = + Deserialize::deserialize(deserializer)?; + + if serializable_format.is_empty() { + return Ok(None); + } + + let mut map = MetadataMap::new(); + + for submap in serializable_format.into_iter() { + for (key, value) in submap.into_iter() { + let key = MetadataKey::from_bytes(key.as_bytes()).unwrap(); + map.append(key, value.parse().unwrap()); + } + } + + Ok(Some(map)) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn serialize_metadata_map() { + let mut map = MetadataMap::new(); + map.append("foo", "bar".parse().unwrap()); + map.append("foo", "baz".parse().unwrap()); + map.append("bar", "foo".parse().unwrap()); + let mut buffer = Vec::new(); + let mut ser = serde_yaml::Serializer::new(&mut buffer); + serialize(&Some(map), &mut ser).unwrap(); + insta::assert_snapshot!(std::str::from_utf8(&buffer).unwrap()); + let de = serde_yaml::Deserializer::from_slice(&buffer); + deserialize(de).unwrap(); + } + } +} diff --git a/apollo-router/src/plugins/telemetry/otlp/grpc.rs b/apollo-router/src/plugins/telemetry/otlp/grpc.rs deleted file mode 100644 index 94e477f0cd..0000000000 --- a/apollo-router/src/plugins/telemetry/otlp/grpc.rs +++ /dev/null @@ -1,126 +0,0 @@ -use super::ExportConfig; -use crate::configuration::{ConfigurationError, TlsConfig}; -use opentelemetry_otlp::WithExportConfig; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::collections::HashMap; -use tonic::metadata::{KeyAndValueRef, MetadataKey, MetadataMap}; - -#[derive(Debug, Clone, Deserialize, Serialize, Default, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct GrpcExporter { - #[serde(flatten)] - pub export_config: ExportConfig, - pub tls_config: Option, - #[serde( - deserialize_with = "header_map_serde::deserialize", - serialize_with = "header_map_serde::serialize", - default - )] - #[schemars(schema_with = "option_metadata_map", default)] - pub metadata: Option, -} - -fn option_metadata_map(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema { - Option::>::json_schema(gen) -} - -impl GrpcExporter { - pub fn exporter(&self) -> Result { - let mut exporter = opentelemetry_otlp::new_exporter().tonic(); - exporter = self.export_config.apply(exporter); - #[allow(unused_variables)] - if let Some(tls_config) = self.tls_config.as_ref() { - #[cfg(feature = "tls")] - { - exporter = exporter.with_tls_config(tls_config.tls_config()?); - } - #[cfg(not(feature = "tls"))] - { - return Err(ConfigurationError::MissingFeature("tls")); - } - } - if let Some(metadata) = self.metadata.clone() { - exporter = exporter.with_metadata(metadata); - } - Ok(exporter) - } - - pub fn exporter_from_env() -> opentelemetry_otlp::TonicExporterBuilder { - let mut exporter = opentelemetry_otlp::new_exporter().tonic(); - exporter = exporter.with_env(); - exporter - } -} - -mod header_map_serde { - use super::*; - - pub(crate) fn serialize(map: &Option, serializer: S) -> Result - where - S: serde::ser::Serializer, - { - if map.as_ref().map(|x| x.is_empty()).unwrap_or(true) { - return serializer.serialize_none(); - } - - let mut serializable_format = - Vec::with_capacity(map.as_ref().map(|x| x.len()).unwrap_or(0)); - - serializable_format.extend(map.iter().flat_map(|x| x.iter()).map(|key_and_value| { - match key_and_value { - KeyAndValueRef::Ascii(key, value) => { - let mut map = HashMap::with_capacity(1); - map.insert(key.as_str(), value.to_str().unwrap()); - map - } - KeyAndValueRef::Binary(_, _) => todo!(), - } - })); - - serializable_format.serialize(serializer) - } - - pub(crate) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: serde::de::Deserializer<'de>, - { - let serializable_format: Vec> = - Deserialize::deserialize(deserializer)?; - - if serializable_format.is_empty() { - return Ok(None); - } - - let mut map = MetadataMap::new(); - - for submap in serializable_format.into_iter() { - for (key, value) in submap.into_iter() { - let key = MetadataKey::from_bytes(key.as_bytes()).unwrap(); - map.append(key, value.parse().unwrap()); - } - } - - Ok(Some(map)) - } - - #[cfg(test)] - mod tests { - use super::*; - - #[test] - fn serialize_metadata_map() { - let mut map = MetadataMap::new(); - map.append("foo", "bar".parse().unwrap()); - map.append("foo", "baz".parse().unwrap()); - map.append("bar", "foo".parse().unwrap()); - let mut buffer = Vec::new(); - let mut ser = serde_yaml::Serializer::new(&mut buffer); - serialize(&Some(map), &mut ser).unwrap(); - insta::assert_snapshot!(std::str::from_utf8(&buffer).unwrap()); - let de = serde_yaml::Deserializer::from_slice(&buffer); - deserialize(de).unwrap(); - } - } -} diff --git a/apollo-router/src/plugins/telemetry/otlp/http.rs b/apollo-router/src/plugins/telemetry/otlp/http.rs deleted file mode 100644 index 3b71579db7..0000000000 --- a/apollo-router/src/plugins/telemetry/otlp/http.rs +++ /dev/null @@ -1,31 +0,0 @@ -use super::ExportConfig; -use crate::configuration::ConfigurationError; -use opentelemetry_otlp::WithExportConfig; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -#[derive(Debug, Clone, Deserialize, Serialize, Default, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct HttpExporter { - #[serde(flatten)] - pub export_config: ExportConfig, - headers: Option>, -} - -impl HttpExporter { - pub fn exporter(&self) -> Result { - let mut exporter = opentelemetry_otlp::new_exporter().http(); - exporter = self.export_config.apply(exporter); - if let Some(headers) = self.headers.clone() { - exporter = exporter.with_headers(headers); - } - Ok(exporter) - } - - pub fn exporter_from_env() -> opentelemetry_otlp::HttpExporterBuilder { - let mut exporter = opentelemetry_otlp::new_exporter().http(); - exporter = exporter.with_env(); - exporter - } -} diff --git a/apollo-router/src/plugins/telemetry/otlp/mod.rs b/apollo-router/src/plugins/telemetry/otlp/mod.rs deleted file mode 100644 index 875e0fafa0..0000000000 --- a/apollo-router/src/plugins/telemetry/otlp/mod.rs +++ /dev/null @@ -1,192 +0,0 @@ -#[cfg(feature = "otlp-grpc")] -mod grpc; -#[cfg(feature = "otlp-http")] -mod http; - -#[cfg(feature = "otlp-grpc")] -pub use self::grpc::*; -#[cfg(feature = "otlp-http")] -pub use self::http::*; -use super::TraceConfig; -use crate::configuration::ConfigurationError; -#[cfg(feature = "otlp-grpc")] -use futures::{Stream, StreamExt}; -use opentelemetry::sdk::metrics::PushController; -#[cfg(feature = "otlp-grpc")] -use opentelemetry::{sdk::metrics::selectors, util::tokio_interval_stream}; -use opentelemetry_otlp::{Protocol, WithExportConfig}; -use schemars::JsonSchema; -use serde::{Deserialize, Deserializer, Serialize}; -use std::time::Duration; -use url::Url; - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields, rename_all = "snake_case")] -pub struct Otlp { - // TODO: in a future iteration we should get rid of tracing and put tracing at the root level cf https://github.com/apollographql/router/issues/683 - pub tracing: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields, rename_all = "snake_case")] -pub struct Tracing { - pub exporter: Exporter, - pub trace_config: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -pub struct Metrics { - #[serde(flatten)] - pub exporter: Exporter, -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields, rename_all = "snake_case")] -pub enum Exporter { - #[cfg(feature = "otlp-grpc")] - Grpc(Option), - #[cfg(feature = "otlp-http")] - Http(Option), -} - -impl Exporter { - pub fn exporter(&self) -> Result { - match &self { - #[cfg(feature = "otlp-grpc")] - Exporter::Grpc(exporter) => Ok(exporter.clone().unwrap_or_default().exporter()?.into()), - #[cfg(feature = "otlp-http")] - Exporter::Http(exporter) => Ok(exporter.clone().unwrap_or_default().exporter()?.into()), - } - } - - pub fn metrics_exporter(&self) -> Result { - match &self { - #[cfg(feature = "otlp-grpc")] - Exporter::Grpc(exporter) => { - let mut export_config = opentelemetry_otlp::ExportConfig::default(); - if let Some(grpc_exporter_cfg) = exporter { - if let Some(endpoint) = &grpc_exporter_cfg.export_config.endpoint { - export_config.endpoint = endpoint.clone().to_string(); - } - if let Some(timeout) = &grpc_exporter_cfg.export_config.timeout { - export_config.timeout = Duration::from_secs(*timeout); - } - if let Some(protocol) = &grpc_exporter_cfg.export_config.protocol { - export_config.protocol = *protocol; - } - } - - let push_ctrl = opentelemetry_otlp::new_pipeline() - .metrics(tokio::spawn, delayed_interval) - .with_exporter( - opentelemetry_otlp::new_exporter() - .tonic() - .with_export_config(export_config), - ) - .with_aggregator_selector(selectors::simple::Selector::Exact) - .build()?; - - Ok(push_ctrl) - } - #[cfg(feature = "otlp-http")] - Exporter::Http(_exporter) => { - // let mut export_config = opentelemetry_otlp::ExportConfig::default(); - // if let Some(http_exporter_cfg) = exporter { - // if let Some(endpoint) = &http_exporter_cfg.export_config.endpoint { - // export_config.endpoint = endpoint.clone().to_string(); - // } - // if let Some(timeout) = &http_exporter_cfg.export_config.timeout { - // export_config.timeout = Duration::from_secs(*timeout); - // } - // if let Some(protocol) = &http_exporter_cfg.export_config.protocol { - // export_config.protocol = *protocol; - // } - // } - - // let push_ctrl = opentelemetry_otlp::new_pipeline() - // .metrics(tokio::spawn, delayed_interval) - // .with_exporter( - // opentelemetry_otlp::new_exporter() - // .http() - // .with_export_config(export_config), - // ) - // .with_aggregator_selector(selectors::simple::Selector::Exact) - // .build()?; - - // Ok(push_ctrl) - // Related to this issue https://github.com/open-telemetry/opentelemetry-rust/issues/772 - unimplemented!("cannot export metrics to http with otlp") - } - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize, Default, JsonSchema)] -#[serde(deny_unknown_fields, rename_all = "snake_case")] -pub struct ExportConfig { - #[serde(deserialize_with = "endpoint_url", default)] - pub endpoint: Option, - - #[schemars(schema_with = "option_protocol_schema", default)] - pub protocol: Option, - pub timeout: Option, -} - -fn option_protocol_schema(gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema { - Option::::json_schema(gen) -} - -//This is a copy of the Otel protocol enum so that ExportConfig can generate json schema. -#[derive(JsonSchema)] -#[allow(dead_code)] -enum ProtocolMirror { - /// GRPC protocol - Grpc, - // HttpJson, - /// HTTP protocol with binary protobuf - HttpBinary, -} - -impl ExportConfig { - pub fn apply(&self, mut exporter: T) -> T { - if let Some(url) = self.endpoint.as_ref() { - exporter = exporter.with_endpoint(url.as_str()); - } - if let Some(protocol) = self.protocol { - exporter = exporter.with_protocol(protocol); - } - if let Some(secs) = self.timeout { - exporter = exporter.with_timeout(Duration::from_secs(secs)); - } - exporter - } -} - -fn endpoint_url<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let mut buf = String::deserialize(deserializer)?; - - // support the case of a IP:port endpoint - if buf.parse::().is_ok() { - buf = format!("https://{}", buf); - } - - let mut url = Url::parse(&buf).map_err(serde::de::Error::custom)?; - - // support the case of 'collector:4317' where url parses 'collector' - // as the scheme instead of the host - if url.host().is_none() && (url.scheme() != "http" || url.scheme() != "https") { - buf = format!("https://{}", buf); - - url = Url::parse(&buf).map_err(serde::de::Error::custom)?; - } - - Ok(Some(url)) -} - -#[cfg(feature = "otlp-grpc")] -fn delayed_interval(duration: Duration) -> impl Stream { - tokio_interval_stream(duration).skip(1) -} diff --git a/apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__otlp__metadata_map_serde__tests__serialize_metadata_map.snap b/apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__otlp__metadata_map_serde__tests__serialize_metadata_map.snap new file mode 100644 index 0000000000..c5cb9929ec --- /dev/null +++ b/apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__otlp__metadata_map_serde__tests__serialize_metadata_map.snap @@ -0,0 +1,9 @@ +--- +source: apollo-router/src/plugins/telemetry/otlp.rs +expression: "std::str::from_utf8(&buffer).unwrap()" +--- +--- +- foo: bar +- foo: baz +- bar: foo + diff --git a/apollo-router/src/plugins/telemetry/tracing/apollo.rs b/apollo-router/src/plugins/telemetry/tracing/apollo.rs new file mode 100644 index 0000000000..d81474348d --- /dev/null +++ b/apollo-router/src/plugins/telemetry/tracing/apollo.rs @@ -0,0 +1,43 @@ +use crate::plugins::telemetry::config::Trace; +use crate::plugins::telemetry::tracing::apollo_telemetry::{SpaceportConfig, StudioGraph}; +use crate::plugins::telemetry::tracing::{apollo_telemetry, TracingConfigurator}; +use opentelemetry::sdk::trace::Builder; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use tower::BoxError; +use url::Url; + +#[derive(Default, Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Config { + pub endpoint: Option, + pub apollo_key: Option, + pub apollo_graph_ref: Option, +} + +impl TracingConfigurator for Config { + fn apply(&self, builder: Builder, trace_config: &Trace) -> Result { + tracing::debug!("configuring Apollo tracing"); + Ok(match self { + Config { + endpoint: Some(endpoint), + apollo_key: Some(key), + apollo_graph_ref: Some(reference), + } => { + tracing::debug!("configuring exporter to Spaceport"); + let exporter = apollo_telemetry::new_pipeline() + .with_trace_config(trace_config.into()) + .with_graph_config(&Some(StudioGraph { + reference: reference.clone(), + key: key.clone(), + })) + .with_spaceport_config(&Some(SpaceportConfig { + collector: endpoint.to_string(), + })) + .build_exporter()?; + builder.with_batch_exporter(exporter, opentelemetry::runtime::Tokio) + } + _ => builder, + }) + } +} diff --git a/apollo-router/src/apollo_telemetry.rs b/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs similarity index 98% rename from apollo-router/src/apollo_telemetry.rs rename to apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs index 82359cf57c..6382ada1ee 100644 --- a/apollo-router/src/apollo_telemetry.rs +++ b/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs @@ -51,26 +51,16 @@ use std::time::Duration; use tokio::task::JoinError; const DEFAULT_SERVER_URL: &str = "https://127.0.0.1:50051"; -const DEFAULT_LISTEN: &str = "127.0.0.1:50051"; fn default_collector() -> String { DEFAULT_SERVER_URL.to_string() } -fn default_listener() -> String { - DEFAULT_LISTEN.to_string() -} - #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] #[serde(deny_unknown_fields, rename_all = "snake_case")] pub struct SpaceportConfig { - pub(crate) external: bool, - #[serde(default = "default_collector")] pub(crate) collector: String, - - #[serde(default = "default_listener")] - pub(crate) listener: String, } #[derive(Clone, Derivative, Deserialize, Serialize, JsonSchema)] @@ -99,8 +89,6 @@ impl Default for SpaceportConfig { fn default() -> Self { Self { collector: default_collector(), - listener: default_listener(), - external: false, } } } @@ -128,6 +116,7 @@ impl Default for PipelineBuilder { } } +#[allow(dead_code)] impl PipelineBuilder { const DEFAULT_BATCH_SIZE: usize = 65_536; const DEFAULT_QUEUE_SIZE: usize = 65_536; @@ -153,7 +142,7 @@ impl PipelineBuilder { /// Install the apollo telemetry exporter pipeline with the recommended defaults. pub fn install_batch(mut self) -> Result { - let exporter = self.get_exporter()?; + let exporter = self.build_exporter()?; // Users can override the default batch and queue sizes, but they can't // set them to be lower than our specified defaults; @@ -233,7 +222,7 @@ impl PipelineBuilder { /// Install the apollo telemetry exporter pipeline with the recommended defaults. #[allow(dead_code)] pub fn install_simple(mut self) -> Result { - let exporter = self.get_exporter()?; + let exporter = self.build_exporter()?; let mut provider_builder = sdk::trace::TracerProvider::builder().with_simple_exporter(exporter); @@ -253,7 +242,7 @@ impl PipelineBuilder { } /// Create a client to talk to our spaceport and return an exporter. - pub fn get_exporter(&self) -> Result { + pub fn build_exporter(&self) -> Result { let collector = match self.spaceport_config.clone() { Some(cfg) => cfg.collector, None => DEFAULT_SERVER_URL.to_string(), @@ -327,6 +316,7 @@ impl From<&StudioGraph> for ReporterGraph { impl SpanExporter for Exporter { /// Export spans to apollo telemetry async fn export(&mut self, batch: Vec) -> ExportResult { + tracing::info!("Exporting batch {}", batch.len()); if self.graph.is_none() { // It's an error to try and export statistics without // graph details. We enforce that elsewhere in the code diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog.rs b/apollo-router/src/plugins/telemetry/tracing/datadog.rs new file mode 100644 index 0000000000..4a428c6c93 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/tracing/datadog.rs @@ -0,0 +1,78 @@ +use crate::plugins::telemetry::config::{GenericWith, Trace}; +use crate::plugins::telemetry::tracing::TracingConfigurator; +use opentelemetry::sdk::trace::Builder; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use tower::BoxError; +use url::Url; + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Config { + pub endpoint: AgentEndpoint, +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case", untagged)] +pub enum AgentEndpoint { + Default(AgentDefault), + Url(Url), +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum AgentDefault { + Default, +} + +impl TracingConfigurator for Config { + fn apply(&self, builder: Builder, trace_config: &Trace) -> Result { + tracing::debug!("configuring Datadog tracing"); + let url = match &self.endpoint { + AgentEndpoint::Default(_) => None, + AgentEndpoint::Url(s) => Some(s), + }; + let exporter = opentelemetry_datadog::new_pipeline() + .with(&url, |b, e| b.with_agent_endpoint(e.to_string())) + .with(&trace_config.service_name, |b, n| b.with_service_name(n)) + .with_trace_config(trace_config.into()) + .build_exporter()?; + Ok(builder.with_batch_exporter(exporter, opentelemetry::runtime::Tokio)) + } +} + +#[cfg(test)] +mod tests { + use crate::plugins::telemetry::tracing::test::run_query; + use opentelemetry::global; + use tower::BoxError; + use tracing::instrument::WithSubscriber; + use tracing_subscriber::layer::SubscriberExt; + use tracing_subscriber::Registry; + + // This test can be run manually from your IDE to help with testing otel + // It is set to ignore by default as Datadog may not be set up + #[ignore] + #[tokio::test(flavor = "multi_thread")] + async fn test_tracing() -> Result<(), BoxError> { + tracing_subscriber::fmt().init(); + + global::set_text_map_propagator(opentelemetry_datadog::DatadogPropagator::new()); + let tracer = opentelemetry_datadog::new_pipeline() + .with_service_name("my_app") + .install_batch(opentelemetry::runtime::Tokio)?; + + // Create a tracing layer with the configured tracer + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + + // Use the tracing subscriber `Registry`, or any other subscriber + // that impls `LookupSpan` + let subscriber = Registry::default().with(telemetry); + + // Trace executed code + run_query().with_subscriber(subscriber).await; + global::shutdown_tracer_provider(); + + Ok(()) + } +} diff --git a/apollo-router/src/plugins/telemetry/tracing/jaeger.rs b/apollo-router/src/plugins/telemetry/tracing/jaeger.rs new file mode 100644 index 0000000000..4c258495ad --- /dev/null +++ b/apollo-router/src/plugins/telemetry/tracing/jaeger.rs @@ -0,0 +1,109 @@ +use crate::plugins::telemetry::config::{GenericWith, Trace}; +use crate::plugins::telemetry::tracing::TracingConfigurator; +use opentelemetry::sdk::trace::Builder; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; +use tower::BoxError; +use url::Url; + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Config { + #[serde(flatten)] + pub endpoint: Endpoint, +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum Endpoint { + Agent { + endpoint: AgentEndpoint, + }, + Collector { + endpoint: Url, + username: Option, + password: Option, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case", untagged)] +pub enum AgentEndpoint { + Default(AgentDefault), + Socket(SocketAddr), +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum AgentDefault { + Default, +} + +impl TracingConfigurator for Config { + fn apply(&self, builder: Builder, trace_config: &Trace) -> Result { + tracing::debug!("configuring Jaeger tracing"); + let exporter = match &self.endpoint { + Endpoint::Agent { endpoint } => { + let socket = match endpoint { + AgentEndpoint::Default(_) => None, + AgentEndpoint::Socket(s) => Some(s), + }; + opentelemetry_jaeger::new_pipeline() + .with_trace_config(trace_config.into()) + .with(&trace_config.service_name, |b, n| b.with_service_name(n)) + .with(&socket, |b, s| b.with_agent_endpoint(s)) + .init_async_exporter(opentelemetry::runtime::Tokio)? + } + Endpoint::Collector { + endpoint, + username, + password, + } => opentelemetry_jaeger::new_pipeline() + .with_trace_config(trace_config.into()) + .with(&trace_config.service_name, |b, n| b.with_service_name(n)) + .with(username, |b, u| b.with_collector_username(u)) + .with(password, |b, p| b.with_collector_password(p)) + .with_collector_endpoint(&endpoint.to_string()) + .init_async_exporter(opentelemetry::runtime::Tokio)?, + }; + + Ok(builder.with_batch_exporter(exporter, opentelemetry::runtime::Tokio)) + } +} + +#[cfg(test)] +mod tests { + use crate::plugins::telemetry::tracing::test::run_query; + use opentelemetry::global; + use tower::BoxError; + use tracing::instrument::WithSubscriber; + use tracing_subscriber::layer::SubscriberExt; + use tracing_subscriber::Registry; + + // This test can be run manually from your IDE to help with testing otel + // It is set to ignore by default as jaeger may not be set up + #[ignore] + #[tokio::test(flavor = "multi_thread")] + async fn test_tracing() -> Result<(), BoxError> { + tracing_subscriber::fmt().init(); + + global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); + let tracer = opentelemetry_jaeger::new_pipeline() + .with_service_name("my_app") + .install_batch(opentelemetry::runtime::Tokio)?; + + // Create a tracing layer with the configured tracer + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + + // Use the tracing subscriber `Registry`, or any other subscriber + // that impls `LookupSpan` + let subscriber = Registry::default().with(telemetry); + + // Trace executed code + run_query().with_subscriber(subscriber).await; + global::shutdown_tracer_provider(); + + Ok(()) + } +} diff --git a/apollo-router/src/plugins/telemetry/tracing/mod.rs b/apollo-router/src/plugins/telemetry/tracing/mod.rs new file mode 100644 index 0000000000..41eb7e63d1 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/tracing/mod.rs @@ -0,0 +1,42 @@ +use crate::plugins::telemetry::config::Trace; +use opentelemetry::sdk::trace::Builder; +use tower::BoxError; + +pub mod apollo; +pub mod apollo_telemetry; +pub mod datadog; +pub mod jaeger; +pub mod otlp; +pub mod zipkin; + +pub trait TracingConfigurator { + fn apply(&self, builder: Builder, trace_config: &Trace) -> Result; +} + +#[cfg(test)] +mod test { + use http::{Method, Request, Uri}; + use opentelemetry::global; + use opentelemetry_http::HttpClient; + use tracing::{info_span, Instrument}; + use tracing_opentelemetry::OpenTelemetrySpanExt; + pub async fn run_query() { + let span = info_span!("client_request"); + let client = reqwest::Client::new(); + + let mut request = Request::builder() + .method(Method::POST) + .uri(Uri::from_static("http://localhost:4000")) + .body(r#"{"query":"query {\n topProducts {\n name\n }\n}","variables":{}}"#.into()) + .unwrap(); + + global::get_text_map_propagator(|propagator| { + propagator.inject_context( + &span.context(), + &mut opentelemetry_http::HeaderInjector(request.headers_mut()), + ) + }); + + client.send(request).instrument(span).await.unwrap(); + } +} diff --git a/apollo-router/src/plugins/telemetry/tracing/otlp.rs b/apollo-router/src/plugins/telemetry/tracing/otlp.rs new file mode 100644 index 0000000000..0fedee3869 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/tracing/otlp.rs @@ -0,0 +1,55 @@ +use crate::plugins::telemetry::config::Trace; +use crate::plugins::telemetry::tracing::TracingConfigurator; +use opentelemetry::sdk::trace::Builder; +use opentelemetry_otlp::SpanExporterBuilder; +use std::result::Result; +use tower::BoxError; + +impl TracingConfigurator for super::super::otlp::Config { + fn apply(&self, builder: Builder, _trace_config: &Trace) -> Result { + tracing::debug!("configuring Otlp tracing"); + let exporter: SpanExporterBuilder = self.exporter()?; + Ok(builder.with_batch_exporter( + exporter.build_span_exporter()?, + opentelemetry::runtime::Tokio, + )) + } +} + +#[cfg(test)] +mod tests { + use crate::plugins::telemetry::tracing::test::run_query; + use opentelemetry::global; + use opentelemetry::sdk::propagation::TraceContextPropagator; + use tower::BoxError; + use tracing::instrument::WithSubscriber; + use tracing_subscriber::layer::SubscriberExt; + use tracing_subscriber::Registry; + + // This test can be run manually from your IDE to help with testing otel + // It is set to ignore by default as otlp may not be set up + #[ignore] + #[tokio::test(flavor = "multi_thread")] + async fn test_tracing() -> Result<(), BoxError> { + tracing_subscriber::fmt().init(); + + global::set_text_map_propagator(TraceContextPropagator::new()); + let tracer = opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(opentelemetry_otlp::new_exporter().http()) + .install_batch(opentelemetry::runtime::Tokio)?; + + // Create a tracing layer with the configured tracer + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + + // Use the tracing subscriber `Registry`, or any other subscriber + // that impls `LookupSpan` + let subscriber = Registry::default().with(telemetry); + + // Trace executed code + run_query().with_subscriber(subscriber).await; + global::shutdown_tracer_provider(); + + Ok(()) + } +} diff --git a/apollo-router/src/plugins/telemetry/otlp/snapshots/apollo_router__plugins__telemetry__otlp__grpc__header_map_serde__tests__serialize_metadata_map.snap b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__otlp__grpc__header_map_serde__tests__serialize_metadata_map.snap similarity index 100% rename from apollo-router/src/plugins/telemetry/otlp/snapshots/apollo_router__plugins__telemetry__otlp__grpc__header_map_serde__tests__serialize_metadata_map.snap rename to apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__otlp__grpc__header_map_serde__tests__serialize_metadata_map.snap diff --git a/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__otlp__grpc__header_map_serde__tests__serialize_metadata_map.snap b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__otlp__grpc__header_map_serde__tests__serialize_metadata_map.snap new file mode 100644 index 0000000000..4b6afed1c5 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__otlp__grpc__header_map_serde__tests__serialize_metadata_map.snap @@ -0,0 +1,10 @@ +--- +source: apollo-router/src/plugins/telemetry/tracing/otlp.rs +assertion_line: 178 +expression: "std::str::from_utf8(&buffer).unwrap()" +--- +--- +- foo: bar +- foo: baz +- bar: foo + diff --git a/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__otlp__header_map_serde__tests__serialize_metadata_map.snap b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__otlp__header_map_serde__tests__serialize_metadata_map.snap new file mode 100644 index 0000000000..06b15e8beb --- /dev/null +++ b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__otlp__header_map_serde__tests__serialize_metadata_map.snap @@ -0,0 +1,9 @@ +--- +source: apollo-router/src/plugins/telemetry/tracing/otlp.rs +expression: "std::str::from_utf8(&buffer).unwrap()" +--- +--- +- foo: bar +- foo: baz +- bar: foo + diff --git a/apollo-router/src/plugins/telemetry/tracing/zipkin.rs b/apollo-router/src/plugins/telemetry/tracing/zipkin.rs new file mode 100644 index 0000000000..09df5c4fd0 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/tracing/zipkin.rs @@ -0,0 +1,96 @@ +use crate::plugins::telemetry::config::{GenericWith, Trace}; +use crate::plugins::telemetry::tracing::TracingConfigurator; +use opentelemetry::sdk::trace::Builder; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::net::SocketAddr; +use tower::BoxError; +use url::Url; + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Config { + #[serde(flatten)] + pub endpoint: Endpoint, +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum Endpoint { + Agent { endpoint: AgentEndpoint }, + Collector { endpoint: Url }, +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case", untagged)] +pub enum AgentEndpoint { + Default(AgentDefault), + Socket(SocketAddr), +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields, rename_all = "snake_case")] +pub enum AgentDefault { + Default, +} + +impl TracingConfigurator for Config { + fn apply(&self, builder: Builder, trace_config: &Trace) -> Result { + tracing::debug!("configuring Zipkin tracing"); + let exporter = match &self.endpoint { + Endpoint::Agent { endpoint } => { + let socket = match endpoint { + AgentEndpoint::Default(_) => None, + AgentEndpoint::Socket(s) => Some(s), + }; + opentelemetry_zipkin::new_pipeline() + .with_trace_config(trace_config.into()) + .with(&trace_config.service_name, |b, n| b.with_service_name(n)) + .with(&socket, |b, s| b.with_service_address(*(*s))) + .init_exporter()? + } + Endpoint::Collector { endpoint } => opentelemetry_zipkin::new_pipeline() + .with_trace_config(trace_config.into()) + .with(&trace_config.service_name, |b, n| b.with_service_name(n)) + .with_collector_endpoint(&endpoint.to_string()) + .init_exporter()?, + }; + Ok(builder.with_batch_exporter(exporter, opentelemetry::runtime::Tokio)) + } +} + +#[cfg(test)] +mod tests { + use crate::plugins::telemetry::tracing::test::run_query; + use opentelemetry::global; + use tower::BoxError; + use tracing::instrument::WithSubscriber; + use tracing_subscriber::layer::SubscriberExt; + use tracing_subscriber::Registry; + + // This test can be run manually from your IDE to help with testing otel + // It is set to ignore by default as zipkin may not be set up + #[ignore] + #[tokio::test(flavor = "multi_thread")] + async fn test_tracing() -> Result<(), BoxError> { + tracing_subscriber::fmt().init(); + + global::set_text_map_propagator(opentelemetry_zipkin::Propagator::new()); + let tracer = opentelemetry_zipkin::new_pipeline() + .with_service_name("my_app") + .install_batch(opentelemetry::runtime::Tokio)?; + + // Create a tracing layer with the configured tracer + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + + // Use the tracing subscriber `Registry`, or any other subscriber + // that impls `LookupSpan` + let subscriber = Registry::default().with(telemetry); + + // Trace executed code + run_query().with_subscriber(subscriber).await; + global::shutdown_tracer_provider(); + + Ok(()) + } +} diff --git a/apollo-router/src/router_factory.rs b/apollo-router/src/router_factory.rs index 3495af876a..cccf5efcfe 100644 --- a/apollo-router/src/router_factory.rs +++ b/apollo-router/src/router_factory.rs @@ -5,6 +5,9 @@ use apollo_router_core::{ }; use apollo_router_core::{prelude::*, Context}; use apollo_router_core::{DynPlugin, TowerSubgraphService}; +use envmnt::types::ExpandOptions; +use envmnt::ExpansionType; +use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; use tower::buffer::Buffer; @@ -88,13 +91,6 @@ impl RouterServiceFactory for YamlRouterServiceFactory { builder = builder.with_dyn_plugin(plugin_name, plugin); } - // This **must** run after: - // - the Reporting plugin is initialized. - // - all configuration errors are checked - // and **before** build() is called. - // - // This is because our tracing configuration is initialized by - // the startup() method of our Reporting plugin. let (pluggable_router_service, plugins) = builder.build().await?; let mut previous_plugins = std::mem::replace(&mut self.plugins, plugins); let service = ServiceBuilder::new().buffered().service( @@ -107,13 +103,25 @@ impl RouterServiceFactory for YamlRouterServiceFactory { .map_response(|response| response.response) .boxed_clone(), ); + + // We're good to go with the new service. Let the plugins know that this is about to happen. + // This is needed so that the Telemetry plugin can swap in the new propagator. + // The alternative is that we introduce another service on Plugin that wraps the request + // as a much earlier stage. + for (_, plugin) in &mut self.plugins { + tracing::debug!("activating plugin {}", plugin.name()); + #[allow(deprecated)] + plugin.activate(); + tracing::debug!("activated plugin {}", plugin.name()); + } + // If we get here, everything is good so shutdown our previous plugins for (_, mut plugin) in previous_plugins.drain(..).rev() { if let Err(err) = plugin.shutdown().await { // If we can't shutdown a plugin, we terminate the router since we can't // assume that it is safe to continue. - tracing::error!("Could not stop plugin: {}, error: {}", plugin.name(), err); - tracing::error!("Terminating router..."); + tracing::error!("could not stop plugin: {}, error: {}", plugin.name(), err); + tracing::error!("terminating router..."); std::process::exit(1); } } @@ -141,7 +149,10 @@ async fn process_plugins( name, configuration ); - match factory.create_instance(configuration) { + + // expand any env variables in the config before processing. + let configuration = expand_env_variables(configuration); + match factory.create_instance(&configuration) { Ok(mut plugin) => { tracing::debug!("starting plugin: {}", name); match plugin.startup().await { @@ -196,6 +207,29 @@ async fn process_plugins( } } +fn expand_env_variables(configuration: &serde_json::Value) -> serde_json::Value { + let mut configuration = configuration.clone(); + visit(&mut configuration); + configuration +} + +fn visit(value: &mut serde_json::Value) { + match value { + Value::String(value) => { + *value = envmnt::expand( + value, + Some( + ExpandOptions::new() + .clone_with_expansion_type(ExpansionType::UnixBracketsWithDefaults), + ), + ); + } + Value::Array(a) => a.iter_mut().for_each(visit), + Value::Object(o) => o.iter_mut().for_each(|(_, v)| visit(v)), + _ => {} + } +} + #[cfg(test)] mod test { use crate::router_factory::RouterServiceFactory; diff --git a/apollo-router/src/state_machine.rs b/apollo-router/src/state_machine.rs index 3ed869d727..cce06af1ff 100644 --- a/apollo-router/src/state_machine.rs +++ b/apollo-router/src/state_machine.rs @@ -196,6 +196,10 @@ where None, ) .await + .map(|s| { + tracing::info!("reloaded"); + s + }) .into_ok_or_err2() } @@ -260,7 +264,7 @@ where .create(configuration.clone(), schema.clone(), None) .await .map_err(|err| { - tracing::error!("Cannot create the router: {}", err); + tracing::error!("cannot create the router: {}", err); Errored(FederatedServerError::ServiceCreationError(err)) })?; @@ -281,7 +285,7 @@ where .create(router.clone(), configuration.clone(), None, plugin_handlers) .await .map_err(|err| { - tracing::error!("Cannot start the router: {}", err); + tracing::error!("cannot start the router: {}", err); Errored(err) })?; @@ -342,7 +346,7 @@ where ) .await .map_err(|err| { - tracing::error!("Cannot start the router: {}", err); + tracing::error!("cannot start the router: {}", err); Errored(err) })?; Ok(Running { @@ -354,7 +358,7 @@ where } Err(err) => { tracing::error!( - "Cannot create new router, keeping previous configuration: {}", + "cannot create new router, keeping previous configuration: {}", err ); Err(Running { @@ -637,7 +641,7 @@ mod tests { .expect_create() .times(1) .in_sequence(&mut seq) - .returning(|_, _, _| Err(BoxError::from("Error"))); + .returning(|_, _, _| Err(BoxError::from("error"))); router_factory .expect_plugins() diff --git a/apollo-router/src/warp_http_server_factory.rs b/apollo-router/src/warp_http_server_factory.rs index 7330bcb1da..4f9bc46cb2 100644 --- a/apollo-router/src/warp_http_server_factory.rs +++ b/apollo-router/src/warp_http_server_factory.rs @@ -11,7 +11,8 @@ use http::uri::Authority; use http::{HeaderValue, Method, Uri}; use hyper::server::conn::Http; use once_cell::sync::Lazy; -use opentelemetry::propagation::Extractor; +use opentelemetry::global; +use opentelemetry::trace::TraceContextExt; use std::collections::HashMap; use std::pin::Pin; use std::str::FromStr; @@ -22,10 +23,9 @@ use tokio::net::TcpListener; use tokio::net::UnixListener; use tokio::sync::Notify; use tower::{BoxError, ServiceBuilder, ServiceExt}; -use tower_http::trace::{DefaultMakeSpan, TraceLayer}; +use tower_http::trace::{DefaultMakeSpan, MakeSpan, TraceLayer}; use tower_service::Service; use tracing::{Level, Span}; -use tracing_opentelemetry::OpenTelemetrySpanExt; use warp::path::FullPath; use warp::{ http::{header::HeaderMap, StatusCode}, @@ -113,15 +113,14 @@ impl HttpServerFactory for WarpHttpServerFactory { )); // generate a hyper service from warp routes - let svc = warp::service(routes); - let svc = ServiceBuilder::new() // generate a tracing span that covers request parsing and response serializing .layer( - TraceLayer::new_for_http() - .make_span_with(DefaultMakeSpan::new().level(Level::INFO)), + TraceLayer::new_for_http().make_span_with(PropagatingMakeSpan( + DefaultMakeSpan::new().level(Level::INFO), + )), ) - .service(svc); + .service(warp::service(routes)); // if we received a TCP listener, reuse it, otherwise create a new one #[cfg_attr(not(unix), allow(unused_mut))] @@ -532,11 +531,6 @@ where ); } - // retrieve and reuse the potential trace id from the caller - opentelemetry::global::get_text_map_propagator(|injector| { - injector.extract_with_context(&Span::current().context(), &HeaderMapCarrier(&header_map)); - }); - async move { match service.ready_oneshot().await { Ok(mut service) => { @@ -599,25 +593,27 @@ fn prefers_html(accept_header: String) -> bool { .any(|a| a == "text/html") } -struct HeaderMapCarrier<'a>(&'a HeaderMap); +#[derive(Clone)] +struct PropagatingMakeSpan(DefaultMakeSpan); -impl<'a> Extractor for HeaderMapCarrier<'a> { - fn get(&self, key: &str) -> Option<&str> { - if let Some(value) = self.0.get(key).and_then(|x| x.to_str().ok()) { - tracing::trace!( - "found OpenTelemetry key in user's request: {}={}", - key, - value - ); - Some(value) +impl MakeSpan for PropagatingMakeSpan { + fn make_span(&mut self, request: &http::Request) -> Span { + // Before we make the span we need to attach span info that may have come in from the request. + let context = global::get_text_map_propagator(|propagator| { + propagator.extract(&opentelemetry_http::HeaderExtractor(request.headers())) + }); + + // If there was no span from the request then it will default to the NOOP span. + // Attaching the NOOP span has the effect of preventing further tracing. + if context.span().span_context().is_valid() { + // We have a valid remote span, attach it to the current thread before creating the root span. + let _context_guard = context.attach(); + self.0.make_span(request) } else { - None + // No remote span, we can go ahead and create the span without context. + self.0.make_span(request) } } - - fn keys(&self) -> Vec<&str> { - self.0.keys().map(|x| x.as_str()).collect() - } } #[cfg(test)] diff --git a/apollo-spaceport/Cargo.toml b/apollo-spaceport/Cargo.toml index 3f10f3ce2f..a852d09939 100644 --- a/apollo-spaceport/Cargo.toml +++ b/apollo-spaceport/Cargo.toml @@ -20,6 +20,7 @@ serde_json = { version = "1.0.79" } sys-info = "0.9.1" tonic = "0.6.2" tokio = { version = "1.17.0", features = ["macros", "rt-multi-thread"] } +tokio-stream = {version="0.1.8", features=["net"]} tracing = "0.1.32" tracing-subscriber = { version = "0.3.10", features = ["env-filter", "json"] } diff --git a/apollo-spaceport/README.md b/apollo-spaceport/README.md index dad3432acc..810cc823af 100644 --- a/apollo-spaceport/README.md +++ b/apollo-spaceport/README.md @@ -12,9 +12,19 @@ re-loaded, so it makes more sense to include this configuration here. There is a new optional section that looks like this: ``` -graph: - key: - reference: @ +telemetry: + # Optional Apollo telemetry configuration. + apollo: + + # Optional external Spaceport URL. + # If not specified an in-process spaceport is used. + endpoint: "https://my-spaceport" + + # Optional Apollo key. If not specified the env variable APOLLO_KEY will be used. + apollo_key: "${APOLLO_KEY}" + + # Optional graphs reference. If not specified the env variable APOLLO_GRAPH_REF will be used. + apollo_graph_ref: "${APOLLO_GRAPH_REF}" ``` ## Design @@ -33,29 +43,15 @@ The spaceport is configured from a new optional configuration section which look like this: ``` -spaceport: - external: false - collector: https://127.0.0.1:50051 - listener: 127.0.0.1:50051 +telemetry: + # Optional Apollo telemetry configuration. + apollo: + + # Optional external Spaceport URL. + # If not specified an in-process spaceport is used. + endpoint: "https://my-spaceport" ``` -(The above values are the defaults, so configuring like this will have the same -results as performing no configuration.) - -### external - -This directs the router to start an internal spaceport (default: false) or to send -statistics to an externally configured spaceport. - -### collector - -This directs the router to send statistics to this configured URL. - -### listener - -This is only used if external spaceport is false, in which case a listening spaceport -is spawned and will listen at the specified address. - ### Components #### ApolloTelemetry diff --git a/apollo-spaceport/src/server.rs b/apollo-spaceport/src/server.rs index 07e348b05b..089a533e3e 100644 --- a/apollo-spaceport/src/server.rs +++ b/apollo-spaceport/src/server.rs @@ -13,14 +13,18 @@ use prost::Message; use prost_types::Timestamp; use reqwest::Client; use std::collections::HashMap; +use std::future::Future; use std::io::Write; use std::net::SocketAddr; +use std::pin::Pin; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::net::TcpListener; use tokio::sync::mpsc::Sender; use tokio::sync::Mutex; use tokio::time::{interval, Duration, MissedTickBehavior}; +use tokio_stream::wrappers::TcpListenerStream; use tonic::transport::{Error, Server}; use tonic::{Request, Response, Status}; @@ -70,6 +74,8 @@ impl StatsOrTrace { /// Accept Traces and Stats from clients and transfer to an Apollo Ingress pub struct ReportSpaceport { + shutdown_signal: Option + Send + Sync>>>, + listener: Option, addr: SocketAddr, // This Map will only have a single entry if used internally from a router. // (because a router can only be serving a single graph) @@ -88,7 +94,13 @@ impl ReportSpaceport { /// /// The spaceport will attempt to make the transfer 5 times before failing. If /// the spaceport fails, the data is discarded. - pub fn new(addr: SocketAddr) -> Self { + pub async fn new( + addr: SocketAddr, + shutdown_signal: Option + Send + Sync>>>, + ) -> Result { + let listener = TcpListener::bind(addr).await?; + let addr = listener.local_addr()?; + // Spawn a task which will check if there are reports to // submit every interval. let graph_usage: GraphUsageMap = Arc::new(Mutex::new(HashMap::new())); @@ -119,20 +131,33 @@ impl ReportSpaceport { }; } }); - Self { + Ok(Self { + shutdown_signal, + listener: Some(listener), addr, graph_usage, tx, total, - } + }) + } + + pub fn address(&self) -> &SocketAddr { + &self.addr } /// Start serving requests. - pub async fn serve(self) -> Result<(), Error> { - let addr = self.addr; + pub async fn serve(mut self) -> Result<(), Error> { + let shutdown_signal = self + .shutdown_signal + .take() + .unwrap_or_else(|| Box::pin(std::future::pending())); + let listener = self + .listener + .take() + .expect("should have allocated listener"); Server::builder() .add_service(ReporterServer::new(self)) - .serve(addr) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown_signal) .await } @@ -196,7 +221,7 @@ impl ReportSpaceport { // - if server error, it may be transient so treat as retry-able // - if ok, return ok if status.is_client_error() { - tracing::error!("Client error reported at ingress: {}", data); + tracing::error!("client error reported at ingress: {}", data); return Err(Status::invalid_argument(data)); } else if status.is_server_error() { tracing::warn!("attempt: {}, could not transfer: {}", i + 1, data); diff --git a/apollo-spaceport/src/spaceport.rs b/apollo-spaceport/src/spaceport.rs index 03b956c8d1..a900757207 100644 --- a/apollo-spaceport/src/spaceport.rs +++ b/apollo-spaceport/src/spaceport.rs @@ -32,7 +32,7 @@ async fn main() -> Result<(), Box> { .json() .init(); tracing::info!("spaceport starting"); - let spaceport = ReportSpaceport::new(args.address); + let spaceport = ReportSpaceport::new(args.address, None).await?; spaceport.serve().await?; Ok(()) diff --git a/dockerfiles/federation-demo/federation-demo b/dockerfiles/federation-demo/federation-demo index aa5299624b..6a55932931 160000 --- a/dockerfiles/federation-demo/federation-demo +++ b/dockerfiles/federation-demo/federation-demo @@ -1 +1 @@ -Subproject commit aa5299624b456bccefcd76ce3082d679be0b113a +Subproject commit 6a559329319f63336546d173edd98cac17f9058c diff --git a/docs/source/config.json b/docs/source/config.json index 699264a03a..64e0750343 100644 --- a/docs/source/config.json +++ b/docs/source/config.json @@ -13,9 +13,11 @@ "CORS": "/configuration/cors", "Logging": "/configuration/logging", "Header propagation": "/configuration/header-propagation", - "OpenTelemetry": "/configuration/opentelemetry", - "Subgraph Error Inclusion": "/configuration/subgraph-error-inclusion", + "Apollo telemetry": "/configuration/apollo-telemetry", + "Metrics": "/configuration/metrics", + "Tracing": "/configuration/tracing", "Traffic shaping": "/configuration/traffic-shaping", + "Subgraph Error Inclusion": "/configuration/subgraph-error-inclusion", "Usage reporting": "/configuration/spaceport" }, "Containerization": { diff --git a/docs/source/configuration/apollo-telemetry.mdx b/docs/source/configuration/apollo-telemetry.mdx new file mode 100644 index 0000000000..7a22568a99 --- /dev/null +++ b/docs/source/configuration/apollo-telemetry.mdx @@ -0,0 +1,49 @@ +--- +title: Apollo telemetry +description: Send usage data to Apollo Studio +--- + +The Apollo Router can transmit usage data to Apollo Studio via a reporting agent called **Spaceport**. By default, Spaceport runs automatically as a component _within_ the Apollo Router. Additional details on its modes of operation are provided below. + +> **Note:** Spaceport is in active development. Its behavior and functionality might change before general availability release. We welcome user feedback both during and after this preview period. + +## Enabling usage reporting + +You can enable usage reporting in the Apollo Router by setting the following environment variables: + +```bash title=".env" +APOLLO_KEY= +APOLLO_GRAPH_REF=@ +``` + +More information on usage reporting is available in the [Studio documentation](/studio/metrics/usage-reporting/). + +## Advanced configuration (not recommended) + +Spaceport can run either as an internal component of a single Apollo Router instance, or as an external resource shared by _multiple_ router instances. + +For the majority of users an internal Spaceport instance is sufficient. + +To connect the Apollo Router to an external spaceport specify the endpoint in the config. + +```yaml title="router.yaml" +telemetry: + # Optional Apollo telemetry configuration. + apollo: + + # Optional external Spaceport URL. + # If not specified an in-process spaceport is used. + endpoint: "https://my-spaceport" + + # Optional Apollo key. If not specified the env variable APOLLO_KEY will be used. + apollo_key: "${APOLLO_KEY}" + + # Optional graphs reference. If not specified the env variable APOLLO_GRAPH_REF will be used. + apollo_graph_ref: "${APOLLO_GRAPH_REF}" + +``` + +## Running Spaceport externally (not recommended) + +Running spaceport as a separate process currently requires building from [source](https://github.com/apollographql/router/tree/main/apollo-spaceport). + diff --git a/docs/source/configuration/metrics.mdx b/docs/source/configuration/metrics.mdx new file mode 100644 index 0000000000..b8eb6c9ab3 --- /dev/null +++ b/docs/source/configuration/metrics.mdx @@ -0,0 +1,70 @@ +--- +title: Metrics in the Apollo Router +description: Collect metrics +--- + +import { Link } from "gatsby"; + +## Collecting Metrics + +### Using Prometheus + +You can use [Prometheus and Grafana](https://prometheus.io/docs/visualization/grafana/) to collect metrics and visualize the router metrics. + +```yaml title="router.yaml" +telemetry: + metrics: + prometheus: + # By setting this endpoint you enable the prometheus exporter + # All our endpoints exposed by plugins are namespaced by the name of the plugin + # Then to access to this prometheus endpoint, the full url path will be `/plugins/apollo.telemetry/prometheus` + enabled: true +``` + +Assuming you are running locally: +1. Run a query against the router. +2. Navigate to [http://localhost:4000/plugins/apollo.telemetry/prometheus](http://localhost:4000/plugins/apollo.telemetry/prometheus) to see something like: + +``` +# HELP http_request_duration_seconds Total number of HTTP requests made. +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{le="0.5"} 1 +http_request_duration_seconds_bucket{le="0.9"} 1 +---SNIP--- +``` + +Note that if you have not run a query against the router you will see a blank page as no metrics will have been generated yet! + +### Using OpenTelemetry Collector + +You may send metrics to [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) for processing and eporting metrics. + +```yaml title="router.yaml" +telemetry: + metrics: + otlp: + # Either 'default' or a URL + endpoint: default + + # Optional protocol. Only grpc is supported currently. + # Setting to http will result in configuration failure. + protocol: grpc + + # Optional Grpc configuration + grpc: + domain_name: "my.domain" + key: + file: "" + # env: "" + ca: + file: "" + # env: "" + cert: + file: "" + # env: "" + metadata: + foo: bar + + # Optional timeout in humatime form + timeout: 2s +``` diff --git a/docs/source/configuration/opentelemetry.mdx b/docs/source/configuration/opentelemetry.mdx deleted file mode 100644 index 68da5f0f63..0000000000 --- a/docs/source/configuration/opentelemetry.mdx +++ /dev/null @@ -1,172 +0,0 @@ ---- -title: OpenTelemetry in the Apollo Router -description: Collect tracing information ---- - -import { Link } from "gatsby"; - -The Apollo Router supports [OpenTelemetry](https://opentelemetry.io/), with exporters for: - -- [Jaeger](https://www.jaegertracing.io/) -- [OpenTelemetry Protocol (OTLP)](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md) over HTTP or gRPC. - -The Apollo Router generates spans that include the various phases of serving a request and associated dependencies. This is useful for showing how response time is affected by: - -- Sub-request response times -- Query shape (sub-request dependencies) -- Apollo Router post-processing - -Span data is sent to a collector such as [Jaeger](https://www.jaegertracing.io/), which can assemble spans into a gantt chart for analysis. - -> To get the most out of distributed tracing, _all_ components in your system should be instrumented. - -## OpenTracing support - -Although OpenTelemetry has superseded OpenTracing, you might need to interact with systems that have not yet moved to OpenTelemetry. - -If you're using OpenTelemetry throughout your systems and just want to view in [Jaeger](https://www.jaegertracing.io/), you don't need to configure this. - -```yaml title="router.yaml" -# ...other configuration... -telemetry: - # Optional if you want to enable opentracing propagation headers to your subgraphs - opentracing: - # "zipkin_b3" and "jaeger" formats are supported - format: "zipkin_b3" - - # ...other telemetry configuration... -``` - -## Using Jaeger - -The Apollo Router can be configured to export tracing data to Jaeger either via an agent or http collector. - -```yaml title="router.yaml" -telemetry: - opentelemetry: - jaeger: - # Optional: if not present, jaeger will use the default UDP agent address - #endpoint: - # address for the UDP agent mode - # incomptable with collector - # agent: "127.0.0.1:1234" - - # URL of the HTTP collector - # collector:" http://example.org" - # the username and password are obtained from the environment variables - # JAEGER_USERNAME and JAEGER_PASSWORD - - # Name of the service used in traces - # defaults to router - service_name: "router" - - trace_config: - # Trace sampling: possible values are `AlwaysOn`, `AlwaysOff`, or - # `TraceIdRatioBased: number`. The ratio sampler takes a value between 0 - # and 1 and decides on trace creation whether it will be recorded. - sampler: - TraceIdRatioBased: 0.42 - max_events_per_span: 1 - max_attributes_per_span: 2 - max_links_per_span: 3 - max_attributes_per_event: 4 - max_attributes_per_link: 5 - - # Trace attributes accept a map of: Bool, Float, Int, String, Bool[], Float[], Int[], String[] - attributes: - str: "a" - int: 1 - float: 1.0 - bool: true - str_arr: - - "a" - - "b" - int_arr: - - 1 - - 2 - float_arr: - - 1.0 - - 2.0 - bool_arr: - - true - - false -``` - -## OpenTelemetry Collector via OTLP - -[OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) is a horizontally scalable collector that can be used to receive, process and export your telemetry data in a pluggable way. - -If you find that the built-in telemetry features of the Apollo Router are missing some desired functionality e.g. [exporting to Kafka](https://opentelemetry.io/docs/collector/configuration/#exporters) then it's worth considering this option. - -```yaml title="router.yaml" -telemetry: - # Metrics configuration - metrics: - exporter: - otlp: - grpc: - # URL of the exporter - endpoint: http://127.0.0.1:4317 - # Possible options: 'Grpc' (HttpBinary is not already supported for metrics) - protocol: Grpc - # timmeout in seconds - timeout: 60 - # Configuration to send traces and metrics to an OpenTelemetry Protocol compatible service - opentelemetry: - otlp: - tracing: - exporter: - # 'http' for OTLP/HTTP, 'grpc' for OTLP/gRPC - grpc: - # URL of the exporter - endpoint: http://example.com - # Possible options: 'Grpc' for GRPC protocol and 'HttpBinary' for HTTP protocol with binary protobuf - protocol: Grpc - # timmeout in seconds - timeout: 60 - trace_config: - # trace sampling: possible values are `AlwaysOn`, `AlwaysOff`, or - # `TraceIdRatioBased: number`. The ratio sampler takes a value between 0 - # and 1 and decides on trace creation whether it will be recorded. - sampler: - TraceIdRatioBased: 0.42 - max_events_per_span: 1 - max_attributes_per_span: 2 - max_links_per_span: 3 - max_attributes_per_event: 4 - max_attributes_per_link: 5 - - # Trace attributes accept a map of: Bool, Float, Int, String, Bool[], Float[], Int[], String[] - attributes: - str: "a" - int: 1 - float: 1.0 - bool: true - str_arr: - - "a" - - "b" - int_arr: - - 1 - - 2 - float_arr: - - 1.0 - - 2.0 - bool_arr: - - true - - false -``` - -## Using Prometheus to collect metrics - -You can use [Prometheus and Grafana](https://prometheus.io/docs/visualization/grafana/) to collect metrics and visualize the router metrics. - -```yaml title="router.yaml" -telemetry: - metrics: - exporter: - prometheus: - # By setting this endpoint you enable the prometheus exporter - # All our endpoints exposed by plugins are namespaced by the name of the plugin - # Then to access to this prometheus endpoint, the full url path will be `/plugins/apollo.telemetry/metrics` - endpoint: "/metrics" -``` diff --git a/docs/source/configuration/overview.mdx b/docs/source/configuration/overview.mdx index 2ebe896838..7ba2ca999c 100644 --- a/docs/source/configuration/overview.mdx +++ b/docs/source/configuration/overview.mdx @@ -259,6 +259,23 @@ plugins: var2: 1 ``` +### Environment variable expansion + +You can reference environment variables directly in your yaml file for any section outside of `server`. + +This is useful for referencing secrets without embedding them directly into the yaml. + +Unix style expansion is used. Either: + +* `${ENV_VAR_NAME}`- Expands to the environment variable `ENV_VAR_NAME`. +* `${ENV_VAR_NAME:some_default}` - Expands to `ENV_VAR_NAME` or `some_default` if the environment variable did not exist. + +Only values may be expanded (not keys): +```yaml {4,8} title="router.yaml" +example: + passord: "${MY_PASSWORD}" +``` + ### Reusing configuration You can reuse parts of your configuration file in multiple places using standard YAML aliasing syntax: diff --git a/docs/source/configuration/spaceport.mdx b/docs/source/configuration/spaceport.mdx deleted file mode 100644 index 4ed57b6e92..0000000000 --- a/docs/source/configuration/spaceport.mdx +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: Apollo Router usage reporting -description: Send usage data to Apollo Studio ---- - -The Apollo Router can transmit usage data to Apollo Studio via a reporting agent called **spaceport**. By default, spaceport runs automatically as a component _within_ the Apollo Router. Additional details on its modes of operation are provided below. - -> **Note:** As with the rest of the Apollo Router, spaceport is in active development. Its behavior and functionality might change before general availability release. We welcome user feedback both during and after this preview period. - -## Enabling usage reporting - -You can enable usage reporting in the Apollo Router by setting the following environment variables: - -```bash title=".env" -APOLLO_KEY= -APOLLO_GRAPH_REF=@ -``` - -More information on usage reporting is available in the [Studio documentation](/studio/metrics/usage-reporting/). - -## Advanced configuration - -Spaceport can run either as an [internal component](#internal-spaceport) of a single Apollo Router instance, or as an [external resource](#external-spaceport) shared by _multiple_ router instances. - -```yaml title="router.yaml" -telemetry: - # Spaceport configuration. These values are the default values if not specified - spaceport: - # By default, Apollo Router runs an internal collector. You can override - # this default behavior by setting `external: true`. If `true`, no reporting - # agent spawns, and the router instead communicates with `collector` below. - external: false -``` - -### Internal spaceport (recommended) - -By default, spaceport runs within a single Apollo Router instance. It requires no additional configuration beyond the setup in [Enabling usage reporting](#enabling-usage-reporting). - -You can optionally configure which address the internal spaceport listens on by setting the `listener` property. This should only be necessary if there's a conflict on the default port that Router chooses (e.g., if running multiple routers or other applications using the same port on the same host), or if it's desirable to change the bind address. - -```yaml title="router.yaml" -telemetry: - # Spaceport configuration. These values are the default values if not specified - spaceport: - # If `external` is `false`, this is the interface and port to listen on. - # Omit otherwise. - listener: 127.0.0.1:50051 -``` - -### External spaceport (advanced) - -**Running an external spaceport is not necessary in most cases.** However, it might be desirable in certain production environments where configuration of sensitive key data or allocation of reporting resources needs to be operated centrally. Under heavier workloads, it can also be beneficial to externalize trace processing to reduce the amount of work that individiual router instances perform. - -To enable the external spaceport, you run an additional Router instance with the _exclusive_ role of collecting and processing Studio data. To configure this, the "collector" should set `external` to `false` and an appropriate `listener` address and port combination. - -Router instances _besides_ the "collector" should set `external` to `true`, _omit_ `listener`, and configure the `collector` property to point to the collector router. - -For help with this feature, please [open a discussion](https://github.com/apollographql/router/discussions). - -```yaml title="router.yaml" -telemetry: - # Spaceport configuration. These values are the default values if not specified - spaceport: - # By default, Apollo Router runs an internal collector. You can override - # this default behavior by setting `external: true`. If `true`, no reporting - # agent spawns, and the router instead communicates with `collector` below. - external: true - - # If `external` is `true`, this should be the location of a spaceport - # that's running external from the Router. Omit otherwise. - collector: https://127.0.0.1:50051 -``` diff --git a/docs/source/configuration/tracing.mdx b/docs/source/configuration/tracing.mdx new file mode 100644 index 0000000000..0b147eda0e --- /dev/null +++ b/docs/source/configuration/tracing.mdx @@ -0,0 +1,191 @@ +--- +title: Tracing in the Apollo Router +description: Collect tracing information +--- + +import { Link } from "gatsby"; + +The Apollo Router supports [OpenTelemetry](https://opentelemetry.io/), with exporters for: + +* [Jaeger](https://www.jaegertracing.io/) +* [Zipkin](https://zipkin.io/) +* [Datadog](https://www.datadoghq.com/) +* [OpenTelemetry Protocol (OTLP)](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md) over HTTP or gRPC. + +The Apollo Router generates spans that include the various phases of serving a request and associated dependencies. This is useful for showing how response time is affected by: + +* Sub-request response times +* Query shape (sub-request dependencies) +* Apollo Router post-processing + +Span data is sent to a collector such as [Jaeger](https://www.jaegertracing.io/), which can assemble spans into a gantt chart for analysis. + +> To get the most out of distributed tracing, _all_ components in your system should be instrumented. + +## Common configuration + +### Trace config +The `trace_config` section contains common configuration that will be used in the for all exporters. It is optional and will fall back on env variables as specified by the [OpenTelemetry spec](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/sdk-environment-variables.md) if `service_name` is not set. + +```yaml title="router.yaml" +telemetry: + tracing: + trace_config: + service_name: "router" + service_namespace: "apollo" + # Optional. Either a float between 0 and 1 or 'always_on' or 'always_off' + sampler: 0.1 + + # Optional. Use a parent based sampler. This enables remote spans help make a decision on if a span is sampeld or not. + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/sdk.md#parentbased + parent_based_sampler: false + + # Optional limits + max_attributes_per_event: 10 + max_attributes_per_link: 10 + max_attributes_per_span: 10 + max_events_per_span: 10 + max_links_per_span: 10 + + # Attributes particular to an exporter that have not + # been explicitly handled in Router configuration. + attributes: + some.config.attribute: "config value" +``` + +If `service_name` is set then environment variables are not used. However, it is possible to embed environment variables into your router config using Unix `${key:default}` syntax. + +If no environment variable is set and service_name is not present then the default of `service_unknown` will be used as per the OpenTelemetry spec. + +### Propagation + +The `propagation` section allows you to configure which propagators are active in addition to ones automatically activated by virtue of using an exporter. + +```yaml title="router.yaml" +telemetry: + tracing: + propagation: + # https://www.w3.org/TR/baggage/ + baggage: false + + # https://www.datadoghq.com/ + datadog: false + + # https://www.jaegertracing.io/ (compliant with opentracing) + jaeger: false + + # https://www.w3.org/TR/trace-context/ + trace_context: false + + # https://zipkin.io/ (compliant with opentracing) + zipkin: false + + jaeger: + .. # Jaeger export configuration, no need to specify Jaeger propagation. +``` +Specifying explicit propagation is generally only required if you are using an exporter that supports multiple trace ID formats. For example OpenTelemetry Collector, Jaeger or OpenTracing compatible exporters. + +## Using Datadog + +The Apollo Router can be configured to connect to either the default agent address or a URL. + +```yaml title="router.yaml" +telemetry: + tracing: + datadog: + # Either 'default' or a URL + endpoint: default +``` + +## Using Jaeger + +The Apollo Router can be configured to export tracing data to Jaeger either via an agent or http collector. + +### Agent config +```yaml title="router.yaml" +telemetry: + tracing: + jaeger: + agent: + # Either 'default' or a socket address + endpoint: default + +``` + +### Collector config +If you are using Kubernetes then you can inject your secrets into configuration via env variable. + +```yaml title="router.yaml" +telemetry: + tracing: + jaeger: + collector: + endpoint: "http://my-jaeger-collector" + username: "${JAEGER_USERNAME}" + password: "${JAEGER_PASSWORD}" +``` + +## OpenTelemetry Collector via OTLP + +[OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) is a horizontally scalable collector that can be used to receive, process and export your telemetry data in a pluggable way. + +If you find that the built-in telemetry features of the Apollo Router are missing some desired functionality e.g. [exporting to Kafka](https://opentelemetry.io/docs/collector/configuration/#exporters) then it's worth considering this option. + +```yaml title="router.yaml" +telemetry: + tracing: + otlp: + # Either 'default' or a URL + endpoint: default + + # Optional protocol (Defaults to grpc) + protocol: grpc + + # Optional Grpc configuration + grpc: + domain_name: "my.domain" + key: + file: "" + # env: "" + ca: + file: "" + # env: "" + cert: + file: "" + # env: "" + metadata: + foo: bar + + # Optional Http configuration + http: + headers: + foo: bar + + # Optional timeout in humatime form + timeout: 2s + +``` + + +## Using Zipkin + +The Apollo Router can be configured to export tracing data to Zipkin either via an agent or http collector. + +### Agent config +```yaml title="router.yaml" +telemetry: + tracing: + zipkin: + agent: + # Either 'default' or a socket address + endpoint: default +``` + +### Collector config +```yaml title="router.yaml" +telemetry: + tracing: + zipkin: + collector: + endpoint: "http://my-zipkin-collector" +``` diff --git a/examples/telemetry/README.md b/examples/telemetry/README.md index f537ae1028..8cfd22096d 100644 --- a/examples/telemetry/README.md +++ b/examples/telemetry/README.md @@ -16,8 +16,5 @@ cargo run -- -s ../graphql/supergraph.graphql -c ./jaeger.router.yaml ```bash cargo run -- -s ../graphql/supergraph.graphql -c ./oltp.router.yaml ``` -## Spaceport -```bash -cargo run -- -s ../graphql/supergraph.graphql -c ./spaceport.router.yaml -``` + diff --git a/examples/telemetry/jaeger.router.yaml b/examples/telemetry/jaeger.router.yaml index c0b7042ceb..40f2155625 100644 --- a/examples/telemetry/jaeger.router.yaml +++ b/examples/telemetry/jaeger.router.yaml @@ -1,4 +1,7 @@ telemetry: - opentelemetry: - jaeger: + tracing: + trace_config: service_name: router + jaeger: + agent: + endpoint: default diff --git a/examples/telemetry/otlp.router.yaml b/examples/telemetry/otlp.router.yaml index 24c23cb61f..cbfa7ad982 100644 --- a/examples/telemetry/otlp.router.yaml +++ b/examples/telemetry/otlp.router.yaml @@ -1,7 +1,6 @@ telemetry: - opentelemetry: + tracing: + trace_config: + service_name: router otlp: - tracing: - exporter: - grpc: - protocol: Grpc + endpoint: default diff --git a/examples/telemetry/spaceport.router.yaml b/examples/telemetry/spaceport.router.yaml deleted file mode 100644 index a6c8fddf02..0000000000 --- a/examples/telemetry/spaceport.router.yaml +++ /dev/null @@ -1,14 +0,0 @@ -telemetry: - spaceport: - # By default, Apollo Router runs an internal collector. You can override - # this default behavior by setting `external: true`. If `true`, no reporting - # agent spawns, and the router instead communicates with `collector` below. - external: false - - # If `external` is `true`, this should be the location of a spaceport - # that's running external from the Router. Omit otherwise. - collector: https://127.0.0.1:50051 - - # If `external` is `false`, this is the interface and port to listen on. - # Omit otherwise. - listener: 127.0.0.1:50051 diff --git a/licenses.html b/licenses.html index ffca8eae9a..177685c4e0 100644 --- a/licenses.html +++ b/licenses.html @@ -44,8 +44,8 @@

Third Party Licenses

Overview of licenses:

    -
  • MIT License (69)
    • -
  • Apache License 2.0 (46)
    • +
  • MIT License (70)
    • +
  • Apache License 2.0 (47)
  • ISC License (8)
  • BSD 3-Clause "New" or "Revised" License (7)
  • Mozilla Public License 2.0 (2)
    • @@ -1069,12 +1069,14 @@

    Used by:

  • clap
  • clap_derive
  • opentelemetry
    • +
  • opentelemetry-datadog
  • opentelemetry-http
  • opentelemetry-jaeger
  • opentelemetry-otlp
  • opentelemetry-otlp
  • opentelemetry-prometheus
  • opentelemetry-semantic-conventions
    • +
  • opentelemetry-zipkin
  • os_str_bytes
  • ryu
  • serde_json_traversal
    • @@ -5268,6 +5270,7 @@

    Used by:

  • dyn-clone
  • either
  • env_logger
    • +
  • envmnt
  • error-chain
  • event-listener
  • failure
    • @@ -5277,6 +5280,7 @@

    Used by:

  • flate2
  • fnv
  • form_urlencoded
    • +
  • fsio
  • futures-lite
  • gimli
  • glob
    • @@ -5288,6 +5292,7 @@

    Used by:

  • hermit-abi
  • hotwatch
  • httparse
    • +
  • humantime-serde
  • hyper-rustls
  • hyper-timeout
  • hyper-tls
    • @@ -5374,6 +5379,7 @@

    Used by:

  • toml
  • triomphe
  • typed-builder
    • +
  • typed-builder
  • unicase
  • unicode-bidi
  • unicode-normalization
    • @@ -8121,6 +8127,217 @@

    Used by:

    // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option. // All files in the project carrying such notice may not be copied, modified, or distributed // except according to those terms. + + +
  • +

    Apache License 2.0

    +

    Used by:

    + + 
    Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
  • @@ -8992,6 +9209,7 @@

    Used by:

    Creative Commons Zero v1.0 Universal

    Used by:

    Creative Commons Legal Code
@@ -10468,6 +10686,35 @@ 
    Used by:
    
 The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.
 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
    +
    • +
  • +

    MIT License

    +

    Used by:

    + + 
    MIT License
+
+Copyright (c) 2017 Evgeny Safronov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
diff --git a/xtask/src/commands/test.rs b/xtask/src/commands/test.rs
index 3c81b9d596..32b6986308 100644
--- a/xtask/src/commands/test.rs
+++ b/xtask/src/commands/test.rs
@@ -4,9 +4,7 @@ use anyhow::{ensure, Result};
 use structopt::StructOpt;
 use xtask::*;
 
-const TEST_DEFAULT_ARGS: &[&str] = &["test", "--locked", "--no-default-features"];
-
-const FEATURE_SETS: &[&[&str]] = &[&["otlp-grpc"], &["otlp-http"], &[]];
+const TEST_DEFAULT_ARGS: &[&str] = &["test", "--locked"];
 
 #[derive(Debug, StructOpt)]
 pub struct Test {
@@ -58,7 +56,6 @@ impl Test {
                         .stdout(Stdio::null())
                         .stderr(Stdio::null())
                         .args(TEST_DEFAULT_ARGS)
-                        .args(FEATURE_SETS[0])
                         .arg("--no-run")
                         .spawn()?,
                 )
@@ -77,13 +74,8 @@ impl Test {
             Box::new((demo, guard))
         };
 
-        for features in FEATURE_SETS {
-            eprintln!("Running tests with features: {}", features.join(","));
-            cargo!(
-                TEST_DEFAULT_ARGS,
-                features.iter().flat_map(|feature| ["--features", feature])
-            );
-        }
+        eprintln!("Running tests");
+        cargo!(TEST_DEFAULT_ARGS);
 
         Ok(())
     }