Merge of #6226

common/tracing/Cargo.toml

@@ -19,6 +19,7 @@ console-subscriber = { version = "0.1.6", optional = true }
 once_cell = "1.12.0"
 opentelemetry = { version = "0.17.0", default-features = false, features = ["trace", "rt-tokio"] }
 opentelemetry-jaeger = { version = "0.16.0", features = ["rt-tokio"] }
+sentry-tracing = "0.27.0"
 tonic = "0.7.2"
 tracing = "0.1.35"
 tracing-appender = "0.2.2"

common/tracing/src/lib.rs

@@ -14,6 +14,9 @@
 
 #![feature(backtrace)]
 
+#[macro_use]
+mod macros;
+
 mod config;
 mod logging;
 mod panic_hook;

common/tracing/src/logging.rs

@@ -20,7 +20,9 @@ use std::sync::Once;
 use once_cell::sync::Lazy;
 use opentelemetry::global;
 use opentelemetry::sdk::propagation::TraceContextPropagator;
+use sentry_tracing::EventFilter;
 use tracing::Event;
+use tracing::Level;
 use tracing::Subscriber;
 use tracing_appender::non_blocking::WorkerGuard;
 use tracing_appender::rolling::RollingFileAppender;
@@ -104,6 +106,25 @@ pub fn init_global_tracing(
         jaeger_layer = Some(tracing_opentelemetry::layer().with_tracer(tracer));
     }
 
+    // Sentry Layer.
+    let mut sentry_layer = None;
+    let bend_sentry_env = env::var("DATABEND_SENTRY_DSN").unwrap_or_else(|_| "".to_string());
+    if !bend_sentry_env.is_empty() {
+        sentry_layer = Some(
+            sentry_tracing::layer()
+                .event_filter(|metadata| match metadata.level() {
+                    &Level::ERROR | &Level::WARN => EventFilter::Event,
+                    &Level::INFO | &Level::DEBUG | &Level::TRACE => EventFilter::Breadcrumb,
+                })
+                .span_filter(|metadata| {
+                    matches!(
+                        metadata.level(),
+                        &Level::ERROR | &Level::WARN | &Level::INFO | &Level::DEBUG
+                    )
+                }),
+        );
+    }
+
     let stdout_layer = if enable_stdout == Some(false) {
         None
     } else {
@@ -119,7 +140,8 @@ pub fn init_global_tracing(
         .with(env_filter)
         .with(JsonStorageLayer)
         .with(file_logging_layer)
-        .with(jaeger_layer);
+        .with(jaeger_layer)
+        .with(sentry_layer);
 
     // For tokio-console
     #[cfg(feature = "console")]

common/tracing/src/macros.rs

@@ -0,0 +1,48 @@
+// Copyright 2022 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// Returns the intended databend semver for Sentry as an `Option<Cow<'static, str>>`.
+///
+/// This can be used with `sentry::ClientOptions` to set the databend semver.
+///
+/// # Examples
+///
+/// ```
+/// # #[macro_use] extern crate common_tracing;
+/// # fn main() {
+/// let _sentry = sentry::init(sentry::ClientOptions {
+///     release: common_tracing::databend_semver!(),
+///     ..Default::default()
+/// });
+/// # }
+/// ```
+#[macro_export]
+macro_rules! databend_semver {
+    () => {{
+        use std::sync::Once;
+        static mut INIT: Once = Once::new();
+        static mut RELEASE: Option<String> = None;
+        unsafe {
+            INIT.call_once(|| {
+                RELEASE = option_env!("CARGO_PKG_NAME").and_then(|name| {
+                    option_env!("VERGEN_GIT_SEMVER").map(|version| format!("{}@{}", name, version))
+                });
+            });
+            RELEASE.as_ref().map(|x| {
+                let release: &'static str = ::std::mem::transmute(x.as_str());
+                ::std::borrow::Cow::Borrowed(release)
+            })
+        }
+    }};
+}

docs/doc/100-faq/20-how-to-tracing.md

@@ -181,7 +181,49 @@ Open http://127.0.0.1:16686/
 
 ![](https://datafuse-1253727613.cos.ap-hongkong.myqcloud.com/jaeger-tracing-show.png)
 
-## Explore and diagnose with tokio-console
+## Error Tracking and Performance Monitoring with Sentry
+
+[Sentry](https://github.com/getsentry/sentry) is a developer-first error tracking and performance monitoring platform that helps developers see what actually matters, solve quicker, and learn continuously about their applications.
+
+### Deploy Sentry
+
+Use the SaaS service provided by [sentry.io](https://sentry.io), or deploy it yourself by following [Self-Hosted Sentry](https://develop.sentry.dev/self-hosted/).
+
+<img src="/img/tracing/sentry-hosted.png"/>
+
+### Create a Project
+
+Create a Rust project in Sentry and get the DSN. The DSN in this example is `http://0c44e65426cb4f87ba059464c0740502@127.0.0.1:9000/5` 。
+
+<img src="/img/tracing/sentry-get-dsn.png"/>
+
+### Start Databend
+
+The following two ways of starting are available.
+
+- **Enable Error Tracking Only**
+
+  This will only use the sentry-log feature, which will help us with error tracking.
+
+  ```bash
+  DATABEND_SENTRY_DSN="<your-sentry-dsn>" ./databend-query
+  ```
+
+  <img src="/img/tracing/sentry-error.png"/>
+
+- **Also Enable Performance Monitoring**
+
+  Setting `SENTRY_TRACES_SAMPLE_RATE` greater than `0.0` will allow sentry to perform trace sampling, which will help set up performance monitoring.
+
+  ```bash
+  DATABEND_SENTRY_DSN="<your-sentry-dsn>" SENTRY_TRACES_SAMPLE_RATE=1.0 LOG_LEVEL=DEBUG ./databend-query
+  ```
+
+  **Note:** Set `SENTRY_TRACES_SAMPLE_RATE` a to lower value in production.
+
+  <img src="/img/tracing/sentry-performance.png"/>
+
+## Explore and Diagnose with tokio-console
 
 [tokio-console](https://github.com/tokio-rs/console) is a diagnostics and debugging tool for asynchronous Rust programs. Make sure you have the tool installed before you use it.
 
@@ -228,12 +270,12 @@ tokio-console # for meta console, http://127.0.0.1:6669
 
 **databend-query**
 
-![query console](images/query-console.png)
+<img src="/img/tracing/query-console.png"/>
 
 **databend-meta**
 
-![meta console](images/meta-console.png)
+<img src="/img/tracing/meta-console.png"/>
 
 **task in console**
 
-![task in console](images/task-in-console.png)
+<img src="/img/tracing/task-in-console.png"/>

metasrv/Cargo.toml

@@ -52,6 +52,7 @@ poem = { version = "1.3.31", features = ["rustls"] }
 prometheus = { version = "0.13.1", features = ["process"] }
 prost = "0.10.4"
 semver = "1.0.10"
+sentry = "0.27.0"
 serde = { version = "1.0.137", features = ["derive"] }
 serde-bridge = "0.0.3"
 serde_json = "1.0.81"

metasrv/src/bin/metasrv.rs

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::env;
 use std::ops::Deref;
 use std::sync::Arc;
 
@@ -39,6 +40,24 @@ async fn main(_global_tracker: Arc<RuntimeTracker>) -> common_exception::Result<
         return Ok(());
     }
 
+    let mut _sentry_guard = None;
+    let bend_sentry_env = env::var("DATABEND_SENTRY_DSN").unwrap_or_else(|_| "".to_string());
+    if !bend_sentry_env.is_empty() {
+        // NOTE: `traces_sample_rate` is 0.0 by default, which disable sentry tracing
+        let traces_sample_rate = env::var("SENTRY_TRACES_SAMPLE_RATE")
+            .ok()
+            .map(|s| {
+                s.parse()
+                    .unwrap_or_else(|_| panic!("`{}` was defined but could not be parsed", s))
+            })
+            .unwrap_or(0.0);
+        _sentry_guard = Some(sentry::init((bend_sentry_env, sentry::ClientOptions {
+            release: common_tracing::databend_semver!(),
+            traces_sample_rate,
+            ..Default::default()
+        })));
+    }
+
     let _guards = init_global_tracing(
         "databend-meta",
         conf.log_dir.as_str(),

query/Cargo.toml

@@ -106,6 +106,7 @@ reqwest = "0.11.11"
 rsa = "0.5.0"
 segment-tree = "2.0.0"
 semver = "1.0.10"
+sentry = "0.27.0"
 serde = { version = "1.0.137", features = ["derive"] }
 serde-bridge = "0.0.3"
 serde_json = "1.0.81"