Trace Tag Replacer: functionality to scrub sensitive data from spans (#…

…111)

.github/CODEOWNERS

@@ -11,3 +11,5 @@ NOTICE              @Datadog/libdatadog
 rustfmt.toml        @Datadog/libdatadog
 README.md           @Datadog/libdatadog
 trace-normalization @Datadog/serverless
+trace-obfuscation   @Datadog/serverless
+trace-protobuf      @Datadog/serverless

Cargo.toml

@@ -12,6 +12,7 @@ members = [
     "tools",
     "tools/cc_utils",
     "trace-normalization",
+    "trace-obfuscation",
     "spawn_worker",
     "tests/spawn_from_lib"
 ]

LICENSE-3rdparty.yml

@@ -1,5 +1,5 @@
 ---
-root_name: "datadog-profiling, ddcommon, datadog-profiling-ffi, ddcommon-ffi, ddtelemetry, ddtelemetry-ffi, tools, cc_utils, datadog-trace-normalization, spawn_worker"
+root_name: "datadog-profiling, ddcommon, datadog-profiling-ffi, ddcommon-ffi, ddtelemetry, ddtelemetry-ffi, tools, cc_utils, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-obfuscation, spawn_worker"
 third_party_libraries:
   - package_name: aho-corasick
     package_version: 0.7.20

tools/docker/Dockerfile.build

@@ -78,12 +78,14 @@ COPY "ddtelemetry-ffi/Cargo.toml" "ddtelemetry-ffi/"
 COPY "profiling/Cargo.toml" "profiling/"
 COPY "profiling-ffi/Cargo.toml" "profiling-ffi/"
 COPY "trace-normalization/Cargo.toml" "trace-normalization/"
+COPY "trace-obfuscation/Cargo.toml" "trace-obfuscation/"
+COPY "trace-protobuf/Cargo.toml" "trace-protobuf/"
 COPY "tools/Cargo.toml" "tools/"
 COPY "tools/cc_utils/Cargo.toml" "tools/cc_utils/"
 COPY "spawn_worker/Cargo.toml" "spawn_worker/"
 COPY "tests/spawn_from_lib/Cargo.toml" "tests/spawn_from_lib/"
 RUN find -name "Cargo.toml" | sed -e s#Cargo.toml#src/lib.rs#g | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs
-RUN echo ddtelemetry/benches/ipc.rs tools/src/bin/dedup_headers.rs ddtelemetry/examples/tm-worker-test.rs | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs
+RUN echo trace-obfuscation/benches/replace_trace_tags_bench.rs ddtelemetry/benches/ipc.rs tools/src/bin/dedup_headers.rs ddtelemetry/examples/tm-worker-test.rs | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs
 
 # cache dependencies
 RUN cargo fetch --locked

trace-normalization/Cargo.toml

@@ -5,8 +5,8 @@ authors = ["David Lee <david.lee@datadoghq.com>"]
 edition = "2021"
 
 [dependencies]
-prost = "0.11.6"
 anyhow = "1.0"
+datadog-trace-protobuf = { path = "../trace-protobuf" }
 
 [dev-dependencies]
 rand = "0.8.5"

trace-normalization/src/lib.rs

@@ -5,10 +5,6 @@
 
 #![deny(clippy::all)]
 
-pub mod pb {
-    include!("./pb/pb.rs");
-}
-
 pub mod normalizer;
 
 pub mod normalize_utils;

trace-normalization/src/normalizer.rs

@@ -4,7 +4,7 @@
 // Datadog, Inc.
 
 use crate::normalize_utils;
-use crate::pb;
+use datadog_trace_protobuf::pb;
 use std::time::SystemTime;
 
 const MAX_TYPE_LEN: usize = 100;
@@ -177,7 +177,7 @@ mod tests {
     use crate::normalize_utils;
     use crate::normalizer;
     use crate::normalizer::DEFAULT_SPAN_NAME;
-    use crate::pb;
+    use datadog_trace_protobuf::pb;
     use rand::Rng;
     use std::collections::HashMap;
     use std::time::SystemTime;

trace-obfuscation/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "datadog-trace-obfuscation"
+version = "2.0.0"
+authors = ["David Lee <david.lee@datadoghq.com>"]
+edition = "2021"
+
+[dependencies]
+anyhow = "1.0"
+regex = "1"
+datadog-trace-protobuf = { path = "../trace-protobuf" }
+
+[dev-dependencies]
+duplicate = "0.4.1"
+criterion = "0.4"
+
+[[bench]]
+name = "replace_trace_tags_bench"
+harness = false

trace-obfuscation/benches/replace_trace_tags_bench.rs

@@ -0,0 +1,59 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present
+// Datadog, Inc.
+
+use std::collections::HashMap;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use datadog_trace_obfuscation::replacer;
+use datadog_trace_protobuf::pb;
+
+fn criterion_benchmark(c: &mut Criterion) {
+    let rules: &[replacer::ReplaceRule] = &replacer::parse_rules_from_string(&[
+        ["http.url", "(token/)([^/]*)", "${1}?"],
+        ["http.url", "guid", "[REDACTED]"],
+        ["*", "(token/)([^/]*)", "${1}?"],
+        ["*", "this", "that"],
+        ["custom.tag", "(/foo/bar/).*", "${1}extra"],
+        ["resource.name", "prod", "stage"],
+    ])
+    .unwrap();
+
+    let span_1 = pb::Span {
+        duration: 10000000,
+        error: 0,
+        resource: "GET /some/raclette".to_string(),
+        service: "django".to_string(),
+        name: "django.controller".to_string(),
+        span_id: 123,
+        start: 1448466874000000000,
+        trace_id: 424242,
+        meta: HashMap::from([
+            ("resource.name".to_string(), "this is prod".to_string()),
+            (
+                "http.url".to_string(),
+                "some/[REDACTED]/token/abcdef/abc".to_string(),
+            ),
+            (
+                "other.url".to_string(),
+                "some/guid/token/abcdef/abc".to_string(),
+            ),
+            ("custom.tag".to_string(), "/foo/bar/foo".to_string()),
+        ]),
+        metrics: HashMap::from([("cheese_weight".to_string(), 100000.0)]),
+        parent_id: 1111,
+        r#type: "http".to_string(),
+        meta_struct: HashMap::new(),
+    };
+
+    let mut trace = [span_1];
+    c.bench_function("replace_trace_tags_bench", |b| {
+        b.iter(|| {
+            replacer::replace_trace_tags(black_box(&mut trace), black_box(rules));
+        })
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

trace-obfuscation/src/lib.rs

@@ -0,0 +1,8 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present
+// Datadog, Inc.
+
+#![deny(clippy::all)]
+
+pub mod replacer;