From 14eb5b20dd0a7861bb0209ac1cbfb59b6d7ca1ea Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 14 Feb 2023 16:26:42 -0800 Subject: [PATCH 01/24] obfuscator init --- Cargo.lock | 8 ++++++++ Cargo.toml | 1 + trace-obfuscation/Cargo.toml | 10 ++++++++++ trace-obfuscation/src/lib.rs | 12 ++++++++++++ trace-obfuscation/src/obfuscator.rs | 10 ++++++++++ 5 files changed, 41 insertions(+) create mode 100644 trace-obfuscation/Cargo.toml create mode 100644 trace-obfuscation/src/lib.rs create mode 100644 trace-obfuscation/src/obfuscator.rs diff --git a/Cargo.lock b/Cargo.lock index 73f8da7b3..a77e880f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -388,6 +388,14 @@ dependencies = [ "rand", ] +[[package]] +name = "datadog-trace-obfuscation" +version = "2.0.0" +dependencies = [ + "anyhow", + "prost", +] + [[package]] name = "ddcommon" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index 13ddbbfbd..2957b1b26 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "ddtelemetry-ffi", "tools", "trace-normalization", + "trace-obfuscation", ] # https://doc.rust-lang.org/cargo/reference/resolver.html#feature-resolver-version-2 resolver = "2" diff --git a/trace-obfuscation/Cargo.toml b/trace-obfuscation/Cargo.toml new file mode 100644 index 000000000..9b6c583b3 --- /dev/null +++ b/trace-obfuscation/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "datadog-trace-obfuscation" +version = "2.0.0" +authors = ["David Lee "] +edition = "2021" + +[dependencies] +prost = "0.11.6" +anyhow = "1.0" + diff --git a/trace-obfuscation/src/lib.rs b/trace-obfuscation/src/lib.rs new file mode 100644 index 000000000..defc1f0f7 --- /dev/null +++ b/trace-obfuscation/src/lib.rs @@ -0,0 +1,12 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present +// Datadog, Inc. + +#![deny(clippy::all)] + +pub mod pb { + include!("../../trace-normalization/src/pb/pb.rs"); +} + +pub mod obfuscator; \ No newline at end of file diff --git a/trace-obfuscation/src/obfuscator.rs b/trace-obfuscation/src/obfuscator.rs new file mode 100644 index 000000000..44743d187 --- /dev/null +++ b/trace-obfuscation/src/obfuscator.rs @@ -0,0 +1,10 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present +// Datadog, Inc. + +use crate::pb; + +pub fn obfuscate_span(s: &mut pb::Span) { + +} \ No newline at end of file From 9d8c7e552cb1e1a413fd311a3b5708b32c44d879 Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 15 Feb 2023 15:12:55 -0800 Subject: [PATCH 02/24] add tag replacer --- Cargo.lock | 2 + trace-obfuscation/Cargo.toml | 3 + trace-obfuscation/src/lib.rs | 2 +- trace-obfuscation/src/obfuscator.rs | 179 +++++++++++++++++++++++++++- 4 files changed, 182 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a77e880f8..4854f0dcf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,7 +393,9 @@ name = "datadog-trace-obfuscation" version = "2.0.0" dependencies = [ "anyhow", + "duplicate", "prost", + "regex", ] [[package]] diff --git a/trace-obfuscation/Cargo.toml b/trace-obfuscation/Cargo.toml index 9b6c583b3..f536c64ba 100644 --- a/trace-obfuscation/Cargo.toml +++ b/trace-obfuscation/Cargo.toml @@ -7,4 +7,7 @@ edition = "2021" [dependencies] prost = "0.11.6" anyhow = "1.0" +regex = "1" +[dev-dependencies] +duplicate = "0.4.1" diff --git a/trace-obfuscation/src/lib.rs b/trace-obfuscation/src/lib.rs index defc1f0f7..e54573430 100644 --- a/trace-obfuscation/src/lib.rs +++ b/trace-obfuscation/src/lib.rs @@ -9,4 +9,4 @@ pub mod pb { include!("../../trace-normalization/src/pb/pb.rs"); } -pub mod obfuscator; \ No newline at end of file +pub mod obfuscator; diff --git a/trace-obfuscation/src/obfuscator.rs b/trace-obfuscation/src/obfuscator.rs index 44743d187..7ac04c1c4 100644 --- a/trace-obfuscation/src/obfuscator.rs +++ b/trace-obfuscation/src/obfuscator.rs @@ -4,7 +4,180 @@ // Datadog, Inc. use crate::pb; +use regex::Regex; -pub fn obfuscate_span(s: &mut pb::Span) { - -} \ No newline at end of file +#[derive(Debug)] +pub struct ReplaceRule { + // name specifies the name of the tag that the replace rule addresses. However, + // some exceptions apply such as: + // • "resource.name" will target the resource + // • "*" will target all tags and the resource + name: String, + + // re holds the regex pattern for matching. + re: regex::Regex, + + // repl specifies the replacement string to be used when Pattern matches. + repl: String, +} + +/// Replaces tag values of all spans within a trace with a given set of rules. +pub fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]) { + for rule in rules { + for span in &mut *trace { + match &rule.name[..] { + "*" => { + for (_, val) in span.meta.iter_mut() { + *val = rule.re.replace_all(val, &rule.repl).to_string(); + } + } + "resource.name" => { + span.resource = rule.re.replace_all(&span.resource, &rule.repl).to_string(); + } + _ => { + if let Some(val) = span.meta.get_mut(&rule.name) { + let replaced_tag = rule.re.replace_all(val, &rule.repl).to_string(); + *val = replaced_tag; + } + } + } + } + } +} + +pub fn parse_rules_from_string(rules: &[[&str; 3]]) -> anyhow::Result> { + let mut vec: Vec = Vec::with_capacity(rules.len()); + + for [name, pattern, repl] in rules { + let compiled_regex = match Regex::new(pattern) { + Ok(res) => res, + Err(err) => { + anyhow::bail!(format!( + "Obfuscator Error: Error while parsing rule: {}", + err + )) + } + }; + vec.push(ReplaceRule { + name: name.to_string(), + re: compiled_regex, + repl: repl.to_string(), + }); + } + Ok(vec) +} + +#[cfg(test)] +mod tests { + + use crate::obfuscator; + use crate::pb; + use duplicate::duplicate_item; + use std::collections::HashMap; + + fn new_test_span_with_tags(tags: HashMap<&str, &str>) -> pb::Span { + let mut span = pb::Span { + duration: 10000000, + error: 0, + resource: "GET /some/raclette".to_string(), + service: "django".to_string(), + name: "django.controller".to_string(), + span_id: 123, + start: 1448466874000000000, + trace_id: 424242, + meta: HashMap::new(), + metrics: HashMap::from([("cheese_weight".to_string(), 100000.0)]), + parent_id: 1111, + r#type: "http".to_string(), + meta_struct: HashMap::new(), + }; + for (key, val) in tags { + match key { + "resource.name" => { + span.resource = val.to_string(); + } + _ => { + span.meta.insert(key.to_string(), val.to_string()); + } + } + } + span + } + + #[duplicate_item( + [ + test_name [test_replace_tags] + rules [&[ + ["http.url", "(token/)([^/]*)", "${1}?"], + ["http.url", "guid", "[REDACTED]"], + ["custom.tag", "(/foo/bar/).*", "${1}extra"], + ]] + input [ + HashMap::from([ + ("http.url", "some/guid/token/abcdef/abc"), + ("custom.tag", "/foo/bar/foo"), + ]) + ] + expected [ + HashMap::from([ + ("http.url", "some/[REDACTED]/token/?/abc"), + ("custom.tag", "/foo/bar/extra"), + ]) + ]; + ] + [ + test_name [test_replace_tags_with_exceptions] + rules [&[ + ["*", "(token/)([^/]*)", "${1}?"], + ["*", "this", "that"], + ["http.url", "guid", "[REDACTED]"], + ["custom.tag", "(/foo/bar/).*", "${1}extra"], + ["resource.name", "prod", "stage"], + ]] + input [ + HashMap::from([ + ("resource.name", "this is prod"), + ("http.url", "some/[REDACTED]/token/abcdef/abc"), + ("other.url", "some/guid/token/abcdef/abc"), + ("custom.tag", "/foo/bar/foo"), + ]) + ] + expected [ + HashMap::from([ + ("resource.name", "this is stage"), + ("http.url", "some/[REDACTED]/token/?/abc"), + ("other.url", "some/guid/token/?/abc"), + ("custom.tag", "/foo/bar/extra"), + ]) + ]; + ] + )] + #[test] + fn test_name() { + let parsed_rules = obfuscator::parse_rules_from_string(rules); + let root_span = new_test_span_with_tags(input); + let child_span = new_test_span_with_tags(input); + let mut trace = [root_span, child_span]; + + obfuscator::replace_trace_tags(&mut trace, &parsed_rules.unwrap()); + + for (key, val) in expected { + match key { + "resource.name" => { + assert_eq!(val, trace[0].resource); + assert_eq!(val, trace[1].resource); + } + _ => { + assert_eq!(val, trace[0].meta.get(key).unwrap()); + assert_eq!(val, trace[1].meta.get(key).unwrap()); + } + } + } + } + + #[test] + fn test_parse_rules_invalid_regex() { + let result = obfuscator::parse_rules_from_string(&[["http.url", ")", "${1}?"]]); + assert!(result.is_err()); + } +} From 107066649d224db279bb44b2e52c1e4e1abc8374 Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 15 Feb 2023 15:34:16 -0800 Subject: [PATCH 03/24] update dockerfile tool --- LICENSE-3rdparty.yml | 2 +- tools/docker/Dockerfile.build | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml index 8b53721f6..e9a90d336 100644 --- a/LICENSE-3rdparty.yml +++ b/LICENSE-3rdparty.yml @@ -1,5 +1,5 @@ --- -root_name: "datadog-profiling, ddcommon, datadog-profiling-ffi, ddcommon-ffi, ddtelemetry, ddtelemetry-ffi, tools" +root_name: "datadog-profiling, ddcommon, datadog-profiling-ffi, ddcommon-ffi, ddtelemetry, ddtelemetry-ffi, tools, datadog-trace-normalization" third_party_libraries: - package_name: aho-corasick package_version: 0.7.20 diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index aa92ba464..7fbc36222 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -78,6 +78,7 @@ COPY "ddtelemetry-ffi/Cargo.toml" "ddtelemetry-ffi/" COPY "profiling/Cargo.toml" "profiling/" COPY "profiling-ffi/Cargo.toml" "profiling-ffi/" COPY "trace-normalization/Cargo.toml" "trace-normalization/" +COPY "trace-obfuscation/Cargo.toml" "trace-obfuscation/" COPY "tools/Cargo.toml" "tools/" RUN find -name "Cargo.toml" | sed -e s#Cargo.toml#src/lib.rs#g | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs RUN echo ddtelemetry/benches/ipc.rs tools/src/bin/dedup_headers.rs ddtelemetry/examples/tm-worker-test.rs | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs From 62a1e8c495a7320b5eb42c31d8552fe67596a230 Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 15 Feb 2023 15:48:42 -0800 Subject: [PATCH 04/24] Create DefaultTraceTagReplacer --- trace-obfuscation/src/obfuscator.rs | 93 ++++++++++++++++------------- 1 file changed, 53 insertions(+), 40 deletions(-) diff --git a/trace-obfuscation/src/obfuscator.rs b/trace-obfuscation/src/obfuscator.rs index 7ac04c1c4..a52946e17 100644 --- a/trace-obfuscation/src/obfuscator.rs +++ b/trace-obfuscation/src/obfuscator.rs @@ -6,6 +6,12 @@ use crate::pb; use regex::Regex; +pub trait TraceTagReplacer { + fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]); + fn parse_rules_from_string(rules: &[[&str; 3]]) -> anyhow::Result>; +} + + #[derive(Debug)] pub struct ReplaceRule { // name specifies the name of the tag that the replace rule addresses. However, @@ -21,60 +27,67 @@ pub struct ReplaceRule { repl: String, } -/// Replaces tag values of all spans within a trace with a given set of rules. -pub fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]) { - for rule in rules { - for span in &mut *trace { - match &rule.name[..] { - "*" => { - for (_, val) in span.meta.iter_mut() { - *val = rule.re.replace_all(val, &rule.repl).to_string(); +struct DefaultTraceTagReplacer {} + +impl TraceTagReplacer for DefaultTraceTagReplacer { + + /// Replaces tag values of all spans within a trace with a given set of rules. + fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]) { + for rule in rules { + for span in &mut *trace { + match &rule.name[..] { + "*" => { + for (_, val) in span.meta.iter_mut() { + *val = rule.re.replace_all(val, &rule.repl).to_string(); + } } - } - "resource.name" => { - span.resource = rule.re.replace_all(&span.resource, &rule.repl).to_string(); - } - _ => { - if let Some(val) = span.meta.get_mut(&rule.name) { - let replaced_tag = rule.re.replace_all(val, &rule.repl).to_string(); - *val = replaced_tag; + "resource.name" => { + span.resource = rule.re.replace_all(&span.resource, &rule.repl).to_string(); + } + _ => { + if let Some(val) = span.meta.get_mut(&rule.name) { + let replaced_tag = rule.re.replace_all(val, &rule.repl).to_string(); + *val = replaced_tag; + } } } } } } -} -pub fn parse_rules_from_string(rules: &[[&str; 3]]) -> anyhow::Result> { - let mut vec: Vec = Vec::with_capacity(rules.len()); - - for [name, pattern, repl] in rules { - let compiled_regex = match Regex::new(pattern) { - Ok(res) => res, - Err(err) => { - anyhow::bail!(format!( - "Obfuscator Error: Error while parsing rule: {}", - err - )) - } - }; - vec.push(ReplaceRule { - name: name.to_string(), - re: compiled_regex, - repl: repl.to_string(), - }); + fn parse_rules_from_string(rules: &[[&str; 3]]) -> anyhow::Result> { + let mut vec: Vec = Vec::with_capacity(rules.len()); + + for [name, pattern, repl] in rules { + let compiled_regex = match Regex::new(pattern) { + Ok(res) => res, + Err(err) => { + anyhow::bail!(format!( + "Obfuscator Error: Error while parsing rule: {}", + err + )) + } + }; + vec.push(ReplaceRule { + name: name.to_string(), + re: compiled_regex, + repl: repl.to_string(), + }); + } + Ok(vec) } - Ok(vec) } #[cfg(test)] mod tests { - use crate::obfuscator; + use crate::obfuscator::DefaultTraceTagReplacer; use crate::pb; use duplicate::duplicate_item; use std::collections::HashMap; + use super::TraceTagReplacer; + fn new_test_span_with_tags(tags: HashMap<&str, &str>) -> pb::Span { let mut span = pb::Span { duration: 10000000, @@ -154,12 +167,12 @@ mod tests { )] #[test] fn test_name() { - let parsed_rules = obfuscator::parse_rules_from_string(rules); + let parsed_rules = DefaultTraceTagReplacer::parse_rules_from_string(rules); let root_span = new_test_span_with_tags(input); let child_span = new_test_span_with_tags(input); let mut trace = [root_span, child_span]; - obfuscator::replace_trace_tags(&mut trace, &parsed_rules.unwrap()); + DefaultTraceTagReplacer::replace_trace_tags(&mut trace, &parsed_rules.unwrap()); for (key, val) in expected { match key { @@ -177,7 +190,7 @@ mod tests { #[test] fn test_parse_rules_invalid_regex() { - let result = obfuscator::parse_rules_from_string(&[["http.url", ")", "${1}?"]]); + let result = DefaultTraceTagReplacer::parse_rules_from_string(&[["http.url", ")", "${1}?"]]); assert!(result.is_err()); } } From b9b3e75149cbf0d113c77505ac2c654e63f402bd Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 15 Feb 2023 17:08:49 -0800 Subject: [PATCH 05/24] use borrowed str over owned String --- trace-obfuscation/src/obfuscator.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/trace-obfuscation/src/obfuscator.rs b/trace-obfuscation/src/obfuscator.rs index a52946e17..f2121c2dd 100644 --- a/trace-obfuscation/src/obfuscator.rs +++ b/trace-obfuscation/src/obfuscator.rs @@ -8,23 +8,23 @@ use regex::Regex; pub trait TraceTagReplacer { fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]); - fn parse_rules_from_string(rules: &[[&str; 3]]) -> anyhow::Result>; + fn parse_rules_from_string<'a>(rules: &'a [[&'a str; 3]]) -> anyhow::Result>>; } #[derive(Debug)] -pub struct ReplaceRule { +pub struct ReplaceRule<'a> { // name specifies the name of the tag that the replace rule addresses. However, // some exceptions apply such as: // • "resource.name" will target the resource // • "*" will target all tags and the resource - name: String, + name: &'a str, // re holds the regex pattern for matching. re: regex::Regex, // repl specifies the replacement string to be used when Pattern matches. - repl: String, + repl: &'a str, } struct DefaultTraceTagReplacer {} @@ -38,15 +38,15 @@ impl TraceTagReplacer for DefaultTraceTagReplacer { match &rule.name[..] { "*" => { for (_, val) in span.meta.iter_mut() { - *val = rule.re.replace_all(val, &rule.repl).to_string(); + *val = rule.re.replace_all(val, rule.repl).to_string(); } } "resource.name" => { - span.resource = rule.re.replace_all(&span.resource, &rule.repl).to_string(); + span.resource = rule.re.replace_all(&span.resource, rule.repl).to_string(); } _ => { - if let Some(val) = span.meta.get_mut(&rule.name) { - let replaced_tag = rule.re.replace_all(val, &rule.repl).to_string(); + if let Some(val) = span.meta.get_mut(rule.name) { + let replaced_tag = rule.re.replace_all(val, rule.repl).to_string(); *val = replaced_tag; } } @@ -55,7 +55,7 @@ impl TraceTagReplacer for DefaultTraceTagReplacer { } } - fn parse_rules_from_string(rules: &[[&str; 3]]) -> anyhow::Result> { + fn parse_rules_from_string<'a>(rules: &'a [[&'a str; 3]]) -> anyhow::Result>> { let mut vec: Vec = Vec::with_capacity(rules.len()); for [name, pattern, repl] in rules { @@ -69,9 +69,9 @@ impl TraceTagReplacer for DefaultTraceTagReplacer { } }; vec.push(ReplaceRule { - name: name.to_string(), + name: name, re: compiled_regex, - repl: repl.to_string(), + repl: repl, }); } Ok(vec) From 32107cfd66998ddbc924305adfa3ddbe7129955b Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 15 Feb 2023 17:21:55 -0800 Subject: [PATCH 06/24] lint --- trace-obfuscation/src/obfuscator.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/trace-obfuscation/src/obfuscator.rs b/trace-obfuscation/src/obfuscator.rs index f2121c2dd..3326d1927 100644 --- a/trace-obfuscation/src/obfuscator.rs +++ b/trace-obfuscation/src/obfuscator.rs @@ -8,10 +8,11 @@ use regex::Regex; pub trait TraceTagReplacer { fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]); - fn parse_rules_from_string<'a>(rules: &'a [[&'a str; 3]]) -> anyhow::Result>>; + fn parse_rules_from_string<'a>( + rules: &'a [[&'a str; 3]], + ) -> anyhow::Result>>; } - #[derive(Debug)] pub struct ReplaceRule<'a> { // name specifies the name of the tag that the replace rule addresses. However, @@ -30,12 +31,11 @@ pub struct ReplaceRule<'a> { struct DefaultTraceTagReplacer {} impl TraceTagReplacer for DefaultTraceTagReplacer { - /// Replaces tag values of all spans within a trace with a given set of rules. fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]) { for rule in rules { for span in &mut *trace { - match &rule.name[..] { + match rule.name { "*" => { for (_, val) in span.meta.iter_mut() { *val = rule.re.replace_all(val, rule.repl).to_string(); @@ -55,7 +55,9 @@ impl TraceTagReplacer for DefaultTraceTagReplacer { } } - fn parse_rules_from_string<'a>(rules: &'a [[&'a str; 3]]) -> anyhow::Result>> { + fn parse_rules_from_string<'a>( + rules: &'a [[&'a str; 3]], + ) -> anyhow::Result>> { let mut vec: Vec = Vec::with_capacity(rules.len()); for [name, pattern, repl] in rules { @@ -69,9 +71,9 @@ impl TraceTagReplacer for DefaultTraceTagReplacer { } }; vec.push(ReplaceRule { - name: name, + name, re: compiled_regex, - repl: repl, + repl, }); } Ok(vec) @@ -190,7 +192,8 @@ mod tests { #[test] fn test_parse_rules_invalid_regex() { - let result = DefaultTraceTagReplacer::parse_rules_from_string(&[["http.url", ")", "${1}?"]]); + let result = + DefaultTraceTagReplacer::parse_rules_from_string(&[["http.url", ")", "${1}?"]]); assert!(result.is_err()); } } From ca6daa92b46cde8c896c6499479f43f2cc878abf Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 15 Feb 2023 17:22:52 -0800 Subject: [PATCH 07/24] Update CODEOWNERS --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6b384de98..1e084d3b3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,3 +11,4 @@ NOTICE @Datadog/libdatadog rustfmt.toml @Datadog/libdatadog README.md @Datadog/libdatadog trace-normalization @Datadog/serverless +trace-obfuscation @Datadog/serverless From 530f3cf41008253de03260ba779a3fbe9461ee13 Mon Sep 17 00:00:00 2001 From: David Lee Date: Thu, 16 Feb 2023 09:02:32 -0800 Subject: [PATCH 08/24] rename obfuscator to replacer --- trace-obfuscation/src/lib.rs | 2 +- trace-obfuscation/src/{obfuscator.rs => replacer.rs} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename trace-obfuscation/src/{obfuscator.rs => replacer.rs} (99%) diff --git a/trace-obfuscation/src/lib.rs b/trace-obfuscation/src/lib.rs index e54573430..772a01465 100644 --- a/trace-obfuscation/src/lib.rs +++ b/trace-obfuscation/src/lib.rs @@ -9,4 +9,4 @@ pub mod pb { include!("../../trace-normalization/src/pb/pb.rs"); } -pub mod obfuscator; +pub mod replacer; diff --git a/trace-obfuscation/src/obfuscator.rs b/trace-obfuscation/src/replacer.rs similarity index 99% rename from trace-obfuscation/src/obfuscator.rs rename to trace-obfuscation/src/replacer.rs index 3326d1927..b801cf717 100644 --- a/trace-obfuscation/src/obfuscator.rs +++ b/trace-obfuscation/src/replacer.rs @@ -83,7 +83,7 @@ impl TraceTagReplacer for DefaultTraceTagReplacer { #[cfg(test)] mod tests { - use crate::obfuscator::DefaultTraceTagReplacer; + use crate::replacer::DefaultTraceTagReplacer; use crate::pb; use duplicate::duplicate_item; use std::collections::HashMap; From b8bb46c9867def181ae59e46d8517967a36aee50 Mon Sep 17 00:00:00 2001 From: David Lee Date: Thu, 16 Feb 2023 09:19:02 -0800 Subject: [PATCH 09/24] lint --- trace-obfuscation/src/replacer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trace-obfuscation/src/replacer.rs b/trace-obfuscation/src/replacer.rs index b801cf717..0b849bdba 100644 --- a/trace-obfuscation/src/replacer.rs +++ b/trace-obfuscation/src/replacer.rs @@ -83,8 +83,8 @@ impl TraceTagReplacer for DefaultTraceTagReplacer { #[cfg(test)] mod tests { - use crate::replacer::DefaultTraceTagReplacer; use crate::pb; + use crate::replacer::DefaultTraceTagReplacer; use duplicate::duplicate_item; use std::collections::HashMap; From 4495bfa4793c78ea1626ff0da6e30db5d727e2f8 Mon Sep 17 00:00:00 2001 From: David Lee Date: Thu, 16 Feb 2023 11:27:07 -0800 Subject: [PATCH 10/24] Update replacer.rs --- trace-obfuscation/src/replacer.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trace-obfuscation/src/replacer.rs b/trace-obfuscation/src/replacer.rs index 0b849bdba..941310299 100644 --- a/trace-obfuscation/src/replacer.rs +++ b/trace-obfuscation/src/replacer.rs @@ -31,7 +31,7 @@ pub struct ReplaceRule<'a> { struct DefaultTraceTagReplacer {} impl TraceTagReplacer for DefaultTraceTagReplacer { - /// Replaces tag values of all spans within a trace with a given set of rules. + /// replace_trace_tags replaces the tag values of all spans within a trace with a given set of rules. fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]) { for rule in rules { for span in &mut *trace { @@ -55,6 +55,9 @@ impl TraceTagReplacer for DefaultTraceTagReplacer { } } + /// parse_rules_from_string takes an array of rules, represented as an array of length 3 arrays + /// holding the tag name, regex pattern, and replacement string as strings. + /// * returns a vec of ReplaceRules fn parse_rules_from_string<'a>( rules: &'a [[&'a str; 3]], ) -> anyhow::Result>> { From 6f4cf14eb9f93099df30cb3cda910f541ad9ef52 Mon Sep 17 00:00:00 2001 From: David Lee Date: Thu, 16 Feb 2023 11:38:57 -0800 Subject: [PATCH 11/24] move parse_rules_from_string out of struct --- trace-obfuscation/src/replacer.rs | 62 +++++++++++++++---------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/trace-obfuscation/src/replacer.rs b/trace-obfuscation/src/replacer.rs index 941310299..2cfe4ad56 100644 --- a/trace-obfuscation/src/replacer.rs +++ b/trace-obfuscation/src/replacer.rs @@ -8,9 +8,6 @@ use regex::Regex; pub trait TraceTagReplacer { fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]); - fn parse_rules_from_string<'a>( - rules: &'a [[&'a str; 3]], - ) -> anyhow::Result>>; } #[derive(Debug)] @@ -54,40 +51,40 @@ impl TraceTagReplacer for DefaultTraceTagReplacer { } } } +} - /// parse_rules_from_string takes an array of rules, represented as an array of length 3 arrays - /// holding the tag name, regex pattern, and replacement string as strings. - /// * returns a vec of ReplaceRules - fn parse_rules_from_string<'a>( - rules: &'a [[&'a str; 3]], - ) -> anyhow::Result>> { - let mut vec: Vec = Vec::with_capacity(rules.len()); - - for [name, pattern, repl] in rules { - let compiled_regex = match Regex::new(pattern) { - Ok(res) => res, - Err(err) => { - anyhow::bail!(format!( - "Obfuscator Error: Error while parsing rule: {}", - err - )) - } - }; - vec.push(ReplaceRule { - name, - re: compiled_regex, - repl, - }); - } - Ok(vec) +/// parse_rules_from_string takes an array of rules, represented as an array of length 3 arrays +/// holding the tag name, regex pattern, and replacement string as strings. +/// * returns a vec of ReplaceRules +pub fn parse_rules_from_string<'a>( + rules: &'a [[&'a str; 3]], +) -> anyhow::Result>> { + let mut vec: Vec = Vec::with_capacity(rules.len()); + + for [name, pattern, repl] in rules { + let compiled_regex = match Regex::new(pattern) { + Ok(res) => res, + Err(err) => { + anyhow::bail!(format!( + "Obfuscator Error: Error while parsing rule: {}", + err + )) + } + }; + vec.push(ReplaceRule { + name, + re: compiled_regex, + repl, + }); } + Ok(vec) } #[cfg(test)] mod tests { use crate::pb; - use crate::replacer::DefaultTraceTagReplacer; + use crate::replacer; use duplicate::duplicate_item; use std::collections::HashMap; @@ -172,12 +169,12 @@ mod tests { )] #[test] fn test_name() { - let parsed_rules = DefaultTraceTagReplacer::parse_rules_from_string(rules); + let parsed_rules = replacer::parse_rules_from_string(rules); let root_span = new_test_span_with_tags(input); let child_span = new_test_span_with_tags(input); let mut trace = [root_span, child_span]; - DefaultTraceTagReplacer::replace_trace_tags(&mut trace, &parsed_rules.unwrap()); + replacer::DefaultTraceTagReplacer::replace_trace_tags(&mut trace, &parsed_rules.unwrap()); for (key, val) in expected { match key { @@ -195,8 +192,7 @@ mod tests { #[test] fn test_parse_rules_invalid_regex() { - let result = - DefaultTraceTagReplacer::parse_rules_from_string(&[["http.url", ")", "${1}?"]]); + let result = replacer::parse_rules_from_string(&[["http.url", ")", "${1}?"]]); assert!(result.is_err()); } } From 7d005153f259bba86e4c314f2528306c66c293fe Mon Sep 17 00:00:00 2001 From: David Lee Date: Thu, 16 Feb 2023 12:52:29 -0800 Subject: [PATCH 12/24] remove trait --- trace-obfuscation/src/replacer.rs | 44 ++++++++++++------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/trace-obfuscation/src/replacer.rs b/trace-obfuscation/src/replacer.rs index 2cfe4ad56..feb27b791 100644 --- a/trace-obfuscation/src/replacer.rs +++ b/trace-obfuscation/src/replacer.rs @@ -6,10 +6,6 @@ use crate::pb; use regex::Regex; -pub trait TraceTagReplacer { - fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]); -} - #[derive(Debug)] pub struct ReplaceRule<'a> { // name specifies the name of the tag that the replace rule addresses. However, @@ -25,27 +21,23 @@ pub struct ReplaceRule<'a> { repl: &'a str, } -struct DefaultTraceTagReplacer {} - -impl TraceTagReplacer for DefaultTraceTagReplacer { - /// replace_trace_tags replaces the tag values of all spans within a trace with a given set of rules. - fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]) { - for rule in rules { - for span in &mut *trace { - match rule.name { - "*" => { - for (_, val) in span.meta.iter_mut() { - *val = rule.re.replace_all(val, rule.repl).to_string(); - } +/// replace_trace_tags replaces the tag values of all spans within a trace with a given set of rules. +pub fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]) { + for rule in rules { + for span in &mut *trace { + match rule.name { + "*" => { + for (_, val) in span.meta.iter_mut() { + *val = rule.re.replace_all(val, rule.repl).to_string(); } - "resource.name" => { - span.resource = rule.re.replace_all(&span.resource, rule.repl).to_string(); - } - _ => { - if let Some(val) = span.meta.get_mut(rule.name) { - let replaced_tag = rule.re.replace_all(val, rule.repl).to_string(); - *val = replaced_tag; - } + } + "resource.name" => { + span.resource = rule.re.replace_all(&span.resource, rule.repl).to_string(); + } + _ => { + if let Some(val) = span.meta.get_mut(rule.name) { + let replaced_tag = rule.re.replace_all(val, rule.repl).to_string(); + *val = replaced_tag; } } } @@ -88,8 +80,6 @@ mod tests { use duplicate::duplicate_item; use std::collections::HashMap; - use super::TraceTagReplacer; - fn new_test_span_with_tags(tags: HashMap<&str, &str>) -> pb::Span { let mut span = pb::Span { duration: 10000000, @@ -174,7 +164,7 @@ mod tests { let child_span = new_test_span_with_tags(input); let mut trace = [root_span, child_span]; - replacer::DefaultTraceTagReplacer::replace_trace_tags(&mut trace, &parsed_rules.unwrap()); + replacer::replace_trace_tags(&mut trace, &parsed_rules.unwrap()); for (key, val) in expected { match key { From 89c1874ad22809793b76e4e99790b32a29ceaade Mon Sep 17 00:00:00 2001 From: David Lee Date: Thu, 16 Feb 2023 13:05:35 -0800 Subject: [PATCH 13/24] comment --- trace-obfuscation/src/replacer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trace-obfuscation/src/replacer.rs b/trace-obfuscation/src/replacer.rs index feb27b791..9544980c7 100644 --- a/trace-obfuscation/src/replacer.rs +++ b/trace-obfuscation/src/replacer.rs @@ -10,8 +10,8 @@ use regex::Regex; pub struct ReplaceRule<'a> { // name specifies the name of the tag that the replace rule addresses. However, // some exceptions apply such as: - // • "resource.name" will target the resource - // • "*" will target all tags and the resource + // * "resource.name" will target the resource + // * "*" will target all tags and the resource name: &'a str, // re holds the regex pattern for matching. From 334410a0485ad68af51cfb3205b4fbba166af4e3 Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 21 Feb 2023 08:51:14 -0800 Subject: [PATCH 14/24] Update LICENSE-3rdparty.yml --- LICENSE-3rdparty.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml index ee22dc06e..ae1cdf13f 100644 --- a/LICENSE-3rdparty.yml +++ b/LICENSE-3rdparty.yml @@ -1,5 +1,5 @@ --- -root_name: "datadog-profiling, ddcommon, datadog-profiling-ffi, ddcommon-ffi, ddtelemetry, ddtelemetry-ffi, tools, cc_utils, datadog-trace-normalization, spawn_worker" +root_name: "datadog-profiling, ddcommon, datadog-profiling-ffi, ddcommon-ffi, ddtelemetry, ddtelemetry-ffi, tools, cc_utils, datadog-trace-normalization, datadog-trace-obfuscation, spawn_worker" third_party_libraries: - package_name: aho-corasick package_version: 0.7.20 From 6bba281acf509aef2e08091cf043bd5f1a9988e4 Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 21 Feb 2023 12:03:07 -0800 Subject: [PATCH 15/24] move trace protobuf definitions into their own crate --- Cargo.lock | 11 +++++++++-- trace-normalization/Cargo.toml | 2 +- trace-normalization/src/lib.rs | 4 ---- trace-normalization/src/normalizer.rs | 4 ++-- trace-obfuscation/Cargo.toml | 2 +- trace-obfuscation/src/lib.rs | 4 ---- trace-obfuscation/src/replacer.rs | 6 +++--- trace-protobuf/Cargo.toml | 8 ++++++++ trace-protobuf/src/lib.rs | 10 ++++++++++ .../src/pb/agent_payload.proto | 0 .../src/pb/descriptor.proto | 0 .../src/pb/gogo.proto | 0 {trace-normalization => trace-protobuf}/src/pb/pb.rs | 0 .../src/pb/span.proto | 0 .../src/pb/tracer_payload.proto | 0 15 files changed, 34 insertions(+), 17 deletions(-) create mode 100644 trace-protobuf/Cargo.toml create mode 100644 trace-protobuf/src/lib.rs rename {trace-normalization => trace-protobuf}/src/pb/agent_payload.proto (100%) rename {trace-normalization => trace-protobuf}/src/pb/descriptor.proto (100%) rename {trace-normalization => trace-protobuf}/src/pb/gogo.proto (100%) rename {trace-normalization => trace-protobuf}/src/pb/pb.rs (100%) rename {trace-normalization => trace-protobuf}/src/pb/span.proto (100%) rename {trace-normalization => trace-protobuf}/src/pb/tracer_payload.proto (100%) diff --git a/Cargo.lock b/Cargo.lock index 8b19d21d3..63f9f064c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -391,8 +391,8 @@ name = "datadog-trace-normalization" version = "2.0.0" dependencies = [ "anyhow", + "datadog-trace-protobuf", "duplicate", - "prost", "rand", ] @@ -401,11 +401,18 @@ name = "datadog-trace-obfuscation" version = "2.0.0" dependencies = [ "anyhow", + "datadog-trace-protobuf", "duplicate", - "prost", "regex", ] +[[package]] +name = "datadog-trace-protobuf" +version = "2.0.0" +dependencies = [ + "prost", +] + [[package]] name = "ddcommon" version = "2.0.0" diff --git a/trace-normalization/Cargo.toml b/trace-normalization/Cargo.toml index a0468c533..e5a7abdba 100644 --- a/trace-normalization/Cargo.toml +++ b/trace-normalization/Cargo.toml @@ -5,8 +5,8 @@ authors = ["David Lee "] edition = "2021" [dependencies] -prost = "0.11.6" anyhow = "1.0" +datadog-trace-protobuf = { path = "../trace-protobuf" } [dev-dependencies] rand = "0.8.5" diff --git a/trace-normalization/src/lib.rs b/trace-normalization/src/lib.rs index 93e5a8825..cda2a44ed 100644 --- a/trace-normalization/src/lib.rs +++ b/trace-normalization/src/lib.rs @@ -5,10 +5,6 @@ #![deny(clippy::all)] -pub mod pb { - include!("./pb/pb.rs"); -} - pub mod normalizer; pub mod normalize_utils; diff --git a/trace-normalization/src/normalizer.rs b/trace-normalization/src/normalizer.rs index 492f211cd..3152a36ea 100644 --- a/trace-normalization/src/normalizer.rs +++ b/trace-normalization/src/normalizer.rs @@ -4,7 +4,7 @@ // Datadog, Inc. use crate::normalize_utils; -use crate::pb; +use datadog_trace_protobuf::pb; use std::time::SystemTime; const MAX_TYPE_LEN: usize = 100; @@ -177,7 +177,7 @@ mod tests { use crate::normalize_utils; use crate::normalizer; use crate::normalizer::DEFAULT_SPAN_NAME; - use crate::pb; + use datadog_trace_protobuf::pb; use rand::Rng; use std::collections::HashMap; use std::time::SystemTime; diff --git a/trace-obfuscation/Cargo.toml b/trace-obfuscation/Cargo.toml index f536c64ba..9c027331b 100644 --- a/trace-obfuscation/Cargo.toml +++ b/trace-obfuscation/Cargo.toml @@ -5,9 +5,9 @@ authors = ["David Lee "] edition = "2021" [dependencies] -prost = "0.11.6" anyhow = "1.0" regex = "1" +datadog-trace-protobuf = { path = "../trace-protobuf" } [dev-dependencies] duplicate = "0.4.1" diff --git a/trace-obfuscation/src/lib.rs b/trace-obfuscation/src/lib.rs index 772a01465..b07fd9888 100644 --- a/trace-obfuscation/src/lib.rs +++ b/trace-obfuscation/src/lib.rs @@ -5,8 +5,4 @@ #![deny(clippy::all)] -pub mod pb { - include!("../../trace-normalization/src/pb/pb.rs"); -} - pub mod replacer; diff --git a/trace-obfuscation/src/replacer.rs b/trace-obfuscation/src/replacer.rs index 9544980c7..c9c55d6e7 100644 --- a/trace-obfuscation/src/replacer.rs +++ b/trace-obfuscation/src/replacer.rs @@ -3,7 +3,7 @@ // developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present // Datadog, Inc. -use crate::pb; +use datadog_trace_protobuf::pb; use regex::Regex; #[derive(Debug)] @@ -24,7 +24,7 @@ pub struct ReplaceRule<'a> { /// replace_trace_tags replaces the tag values of all spans within a trace with a given set of rules. pub fn replace_trace_tags(trace: &mut [pb::Span], rules: &[ReplaceRule]) { for rule in rules { - for span in &mut *trace { + for span in trace.iter_mut() { match rule.name { "*" => { for (_, val) in span.meta.iter_mut() { @@ -75,8 +75,8 @@ pub fn parse_rules_from_string<'a>( #[cfg(test)] mod tests { - use crate::pb; use crate::replacer; + use datadog_trace_protobuf::pb; use duplicate::duplicate_item; use std::collections::HashMap; diff --git a/trace-protobuf/Cargo.toml b/trace-protobuf/Cargo.toml new file mode 100644 index 000000000..da06e0076 --- /dev/null +++ b/trace-protobuf/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "datadog-trace-protobuf" +version = "2.0.0" +authors = ["David Lee "] +edition = "2021" + +[dependencies] +prost = "0.11.6" diff --git a/trace-protobuf/src/lib.rs b/trace-protobuf/src/lib.rs new file mode 100644 index 000000000..b07eceea6 --- /dev/null +++ b/trace-protobuf/src/lib.rs @@ -0,0 +1,10 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present +// Datadog, Inc. + +#![deny(clippy::all)] + +pub mod pb { + include!("./pb/pb.rs"); +} diff --git a/trace-normalization/src/pb/agent_payload.proto b/trace-protobuf/src/pb/agent_payload.proto similarity index 100% rename from trace-normalization/src/pb/agent_payload.proto rename to trace-protobuf/src/pb/agent_payload.proto diff --git a/trace-normalization/src/pb/descriptor.proto b/trace-protobuf/src/pb/descriptor.proto similarity index 100% rename from trace-normalization/src/pb/descriptor.proto rename to trace-protobuf/src/pb/descriptor.proto diff --git a/trace-normalization/src/pb/gogo.proto b/trace-protobuf/src/pb/gogo.proto similarity index 100% rename from trace-normalization/src/pb/gogo.proto rename to trace-protobuf/src/pb/gogo.proto diff --git a/trace-normalization/src/pb/pb.rs b/trace-protobuf/src/pb/pb.rs similarity index 100% rename from trace-normalization/src/pb/pb.rs rename to trace-protobuf/src/pb/pb.rs diff --git a/trace-normalization/src/pb/span.proto b/trace-protobuf/src/pb/span.proto similarity index 100% rename from trace-normalization/src/pb/span.proto rename to trace-protobuf/src/pb/span.proto diff --git a/trace-normalization/src/pb/tracer_payload.proto b/trace-protobuf/src/pb/tracer_payload.proto similarity index 100% rename from trace-normalization/src/pb/tracer_payload.proto rename to trace-protobuf/src/pb/tracer_payload.proto From e577cc91ce20e977959aa9dd07c2b951fed4ddc2 Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 21 Feb 2023 12:28:40 -0800 Subject: [PATCH 16/24] Update Dockerfile.build --- tools/docker/Dockerfile.build | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index 559ba1237..45277ac90 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -79,6 +79,7 @@ COPY "profiling/Cargo.toml" "profiling/" COPY "profiling-ffi/Cargo.toml" "profiling-ffi/" COPY "trace-normalization/Cargo.toml" "trace-normalization/" COPY "trace-obfuscation/Cargo.toml" "trace-obfuscation/" +COPY "trace-protobuf/Cargo.toml" "trace-protobuf/" COPY "tools/Cargo.toml" "tools/" COPY "tools/cc_utils/Cargo.toml" "tools/cc_utils/" COPY "spawn_worker/Cargo.toml" "spawn_worker/" From 4dcae5c921866e1602b47ee1b42dcfe4346e9a4f Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 21 Feb 2023 12:29:18 -0800 Subject: [PATCH 17/24] Update CODEOWNERS --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1e084d3b3..54a6fd350 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,3 +12,4 @@ rustfmt.toml @Datadog/libdatadog README.md @Datadog/libdatadog trace-normalization @Datadog/serverless trace-obfuscation @Datadog/serverless +trace-protobuf @Datadog/serverless From 8627022bcd0ad9606832286f8f6871e90ba5ef0e Mon Sep 17 00:00:00 2001 From: David Lee Date: Tue, 21 Feb 2023 12:37:01 -0800 Subject: [PATCH 18/24] Update LICENSE-3rdparty.yml --- LICENSE-3rdparty.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml index ae1cdf13f..c8de9c8f6 100644 --- a/LICENSE-3rdparty.yml +++ b/LICENSE-3rdparty.yml @@ -1,5 +1,5 @@ --- -root_name: "datadog-profiling, ddcommon, datadog-profiling-ffi, ddcommon-ffi, ddtelemetry, ddtelemetry-ffi, tools, cc_utils, datadog-trace-normalization, datadog-trace-obfuscation, spawn_worker" +root_name: "datadog-profiling, ddcommon, datadog-profiling-ffi, ddcommon-ffi, ddtelemetry, ddtelemetry-ffi, tools, cc_utils, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-obfuscation, spawn_worker" third_party_libraries: - package_name: aho-corasick package_version: 0.7.20 From 9ceccb0eab4a0e9c5b169e7620a4d2ea22d73c00 Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 22 Feb 2023 10:42:45 -0800 Subject: [PATCH 19/24] address nit --- trace-obfuscation/src/replacer.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/trace-obfuscation/src/replacer.rs b/trace-obfuscation/src/replacer.rs index c9c55d6e7..f7b713580 100644 --- a/trace-obfuscation/src/replacer.rs +++ b/trace-obfuscation/src/replacer.rs @@ -57,10 +57,7 @@ pub fn parse_rules_from_string<'a>( let compiled_regex = match Regex::new(pattern) { Ok(res) => res, Err(err) => { - anyhow::bail!(format!( - "Obfuscator Error: Error while parsing rule: {}", - err - )) + anyhow::bail!("Obfuscator Error: Error while parsing rule: {}", err) } }; vec.push(ReplaceRule { From c05cccf0f642c249db90ef1fd804b15c6acacc61 Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 22 Feb 2023 16:28:40 -0800 Subject: [PATCH 20/24] add criterion benchmark --- Cargo.lock | 111 +++++++++++++++++- trace-obfuscation/Cargo.toml | 5 + .../benches/replace_trace_tags_benchmark.rs | 48 ++++++++ 3 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 trace-obfuscation/benches/replace_trace_tags_benchmark.rs diff --git a/Cargo.lock b/Cargo.lock index 63f9f064c..e58ae9cd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anyhow" version = "1.0.68" @@ -134,6 +140,33 @@ dependencies = [ "winapi", ] +[[package]] +name = "ciborium" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" + +[[package]] +name = "ciborium-ll" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "2.34.0" @@ -141,10 +174,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" dependencies = [ "bitflags", - "textwrap", + "textwrap 0.11.0", "unicode-width", ] +[[package]] +name = "clap" +version = "3.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +dependencies = [ + "bitflags", + "clap_lex", + "indexmap", + "textwrap 0.16.0", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -195,8 +249,8 @@ checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" dependencies = [ "atty", "cast", - "clap", - "criterion-plot", + "clap 2.34.0", + "criterion-plot 0.4.5", "csv", "itertools", "lazy_static", @@ -213,6 +267,32 @@ dependencies = [ "walkdir", ] +[[package]] +name = "criterion" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +dependencies = [ + "anes", + "atty", + "cast", + "ciborium", + "clap 3.2.23", + "criterion-plot 0.5.0", + "itertools", + "lazy_static", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + [[package]] name = "criterion-plot" version = "0.4.5" @@ -223,6 +303,16 @@ dependencies = [ "itertools", ] +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -401,6 +491,7 @@ name = "datadog-trace-obfuscation" version = "2.0.0" dependencies = [ "anyhow", + "criterion 0.4.0", "datadog-trace-protobuf", "duplicate", "regex", @@ -451,7 +542,7 @@ version = "2.0.0" dependencies = [ "anyhow", "bytes", - "criterion", + "criterion 0.3.6", "ddcommon", "futures", "http", @@ -1134,6 +1225,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "os_str_bytes" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" + [[package]] name = "output_vt100" version = "0.1.3" @@ -1788,6 +1885,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "textwrap" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" + [[package]] name = "thiserror" version = "1.0.38" diff --git a/trace-obfuscation/Cargo.toml b/trace-obfuscation/Cargo.toml index 9c027331b..9bc82028c 100644 --- a/trace-obfuscation/Cargo.toml +++ b/trace-obfuscation/Cargo.toml @@ -11,3 +11,8 @@ datadog-trace-protobuf = { path = "../trace-protobuf" } [dev-dependencies] duplicate = "0.4.1" +criterion = "0.4" + +[[bench]] +name = "replace_trace_tags_benchmark" +harness = false \ No newline at end of file diff --git a/trace-obfuscation/benches/replace_trace_tags_benchmark.rs b/trace-obfuscation/benches/replace_trace_tags_benchmark.rs new file mode 100644 index 000000000..1e6490657 --- /dev/null +++ b/trace-obfuscation/benches/replace_trace_tags_benchmark.rs @@ -0,0 +1,48 @@ +use std::collections::HashMap; + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use datadog_trace_obfuscation::replacer; +use datadog_trace_protobuf::pb; + +fn criterion_benchmark(c: &mut Criterion) { + + let rules: &[replacer::ReplaceRule] = &replacer::parse_rules_from_string( +&[ + ["http.url", "(token/)([^/]*)", "${1}?"], + ["http.url", "guid", "[REDACTED]"], + ["*", "(token/)([^/]*)", "${1}?"], + ["*", "this", "that"], + ["custom.tag", "(/foo/bar/).*", "${1}extra"], + ["resource.name", "prod", "stage"], + ] + ).unwrap(); + + let span_1 = pb::Span { + duration: 10000000, + error: 0, + resource: "GET /some/raclette".to_string(), + service: "django".to_string(), + name: "django.controller".to_string(), + span_id: 123, + start: 1448466874000000000, + trace_id: 424242, + meta: HashMap::from([ + ("resource.name".to_string(), "this is prod".to_string()), + ("http.url".to_string(), "some/[REDACTED]/token/abcdef/abc".to_string()), + ("other.url".to_string(), "some/guid/token/abcdef/abc".to_string()), + ("custom.tag".to_string(), "/foo/bar/foo".to_string()), + ]), + metrics: HashMap::from([("cheese_weight".to_string(), 100000.0)]), + parent_id: 1111, + r#type: "http".to_string(), + meta_struct: HashMap::new(), + }; + + let mut trace = [span_1]; + c.bench_function("replace_trace_tags_bench", |b| b.iter(|| { + replacer::replace_trace_tags(black_box(&mut trace), black_box(rules)); + })); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); \ No newline at end of file From 0a28c71e045f8c4e2af33f3615fe9bba000305ce Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 22 Feb 2023 16:31:59 -0800 Subject: [PATCH 21/24] lint --- .../benches/replace_trace_tags_benchmark.rs | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/trace-obfuscation/benches/replace_trace_tags_benchmark.rs b/trace-obfuscation/benches/replace_trace_tags_benchmark.rs index 1e6490657..9ee62296b 100644 --- a/trace-obfuscation/benches/replace_trace_tags_benchmark.rs +++ b/trace-obfuscation/benches/replace_trace_tags_benchmark.rs @@ -5,17 +5,15 @@ use datadog_trace_obfuscation::replacer; use datadog_trace_protobuf::pb; fn criterion_benchmark(c: &mut Criterion) { - - let rules: &[replacer::ReplaceRule] = &replacer::parse_rules_from_string( -&[ - ["http.url", "(token/)([^/]*)", "${1}?"], - ["http.url", "guid", "[REDACTED]"], - ["*", "(token/)([^/]*)", "${1}?"], - ["*", "this", "that"], - ["custom.tag", "(/foo/bar/).*", "${1}extra"], - ["resource.name", "prod", "stage"], - ] - ).unwrap(); + let rules: &[replacer::ReplaceRule] = &replacer::parse_rules_from_string(&[ + ["http.url", "(token/)([^/]*)", "${1}?"], + ["http.url", "guid", "[REDACTED]"], + ["*", "(token/)([^/]*)", "${1}?"], + ["*", "this", "that"], + ["custom.tag", "(/foo/bar/).*", "${1}extra"], + ["resource.name", "prod", "stage"], + ]) + .unwrap(); let span_1 = pb::Span { duration: 10000000, @@ -28,8 +26,14 @@ fn criterion_benchmark(c: &mut Criterion) { trace_id: 424242, meta: HashMap::from([ ("resource.name".to_string(), "this is prod".to_string()), - ("http.url".to_string(), "some/[REDACTED]/token/abcdef/abc".to_string()), - ("other.url".to_string(), "some/guid/token/abcdef/abc".to_string()), + ( + "http.url".to_string(), + "some/[REDACTED]/token/abcdef/abc".to_string(), + ), + ( + "other.url".to_string(), + "some/guid/token/abcdef/abc".to_string(), + ), ("custom.tag".to_string(), "/foo/bar/foo".to_string()), ]), metrics: HashMap::from([("cheese_weight".to_string(), 100000.0)]), @@ -39,10 +43,12 @@ fn criterion_benchmark(c: &mut Criterion) { }; let mut trace = [span_1]; - c.bench_function("replace_trace_tags_bench", |b| b.iter(|| { - replacer::replace_trace_tags(black_box(&mut trace), black_box(rules)); - })); + c.bench_function("replace_trace_tags_bench", |b| { + b.iter(|| { + replacer::replace_trace_tags(black_box(&mut trace), black_box(rules)); + }) + }); } criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); \ No newline at end of file +criterion_main!(benches); From baf1c55bd36b58e6dcee8c7736030c11a89f96b2 Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 22 Feb 2023 16:47:12 -0800 Subject: [PATCH 22/24] fix CI --- tools/docker/Dockerfile.build | 1 + trace-obfuscation/benches/replace_trace_tags_benchmark.rs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index 45277ac90..5994fde27 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -79,6 +79,7 @@ COPY "profiling/Cargo.toml" "profiling/" COPY "profiling-ffi/Cargo.toml" "profiling-ffi/" COPY "trace-normalization/Cargo.toml" "trace-normalization/" COPY "trace-obfuscation/Cargo.toml" "trace-obfuscation/" +COPY "trace-obfuscation/benches/replace_trace_tags_benchmark.rs" "trace-obfuscation/benches" COPY "trace-protobuf/Cargo.toml" "trace-protobuf/" COPY "tools/Cargo.toml" "tools/" COPY "tools/cc_utils/Cargo.toml" "tools/cc_utils/" diff --git a/trace-obfuscation/benches/replace_trace_tags_benchmark.rs b/trace-obfuscation/benches/replace_trace_tags_benchmark.rs index 9ee62296b..a6c3d8266 100644 --- a/trace-obfuscation/benches/replace_trace_tags_benchmark.rs +++ b/trace-obfuscation/benches/replace_trace_tags_benchmark.rs @@ -1,3 +1,8 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present +// Datadog, Inc. + use std::collections::HashMap; use criterion::{black_box, criterion_group, criterion_main, Criterion}; From a4de7ad8f99974e29f9d0368616d71aeeb337277 Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 22 Feb 2023 16:53:30 -0800 Subject: [PATCH 23/24] Update Dockerfile.build --- tools/docker/Dockerfile.build | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index 5994fde27..7478efb1f 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -79,14 +79,13 @@ COPY "profiling/Cargo.toml" "profiling/" COPY "profiling-ffi/Cargo.toml" "profiling-ffi/" COPY "trace-normalization/Cargo.toml" "trace-normalization/" COPY "trace-obfuscation/Cargo.toml" "trace-obfuscation/" -COPY "trace-obfuscation/benches/replace_trace_tags_benchmark.rs" "trace-obfuscation/benches" COPY "trace-protobuf/Cargo.toml" "trace-protobuf/" COPY "tools/Cargo.toml" "tools/" COPY "tools/cc_utils/Cargo.toml" "tools/cc_utils/" COPY "spawn_worker/Cargo.toml" "spawn_worker/" COPY "tests/spawn_from_lib/Cargo.toml" "tests/spawn_from_lib/" RUN find -name "Cargo.toml" | sed -e s#Cargo.toml#src/lib.rs#g | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs -RUN echo ddtelemetry/benches/ipc.rs tools/src/bin/dedup_headers.rs ddtelemetry/examples/tm-worker-test.rs | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs +RUN echo trace-obfuscation/benches/replace_trace_tags_benchmark.rs ddtelemetry/benches/ipc.rs tools/src/bin/dedup_headers.rs ddtelemetry/examples/tm-worker-test.rs | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs # cache dependencies RUN cargo fetch --locked From 21666454d613f1a8a5195c958db7332ec2264090 Mon Sep 17 00:00:00 2001 From: David Lee Date: Wed, 22 Feb 2023 17:10:58 -0800 Subject: [PATCH 24/24] rename bench file --- tools/docker/Dockerfile.build | 2 +- trace-obfuscation/Cargo.toml | 2 +- ...lace_trace_tags_benchmark.rs => replace_trace_tags_bench.rs} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename trace-obfuscation/benches/{replace_trace_tags_benchmark.rs => replace_trace_tags_bench.rs} (100%) diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index 7478efb1f..693d9ce43 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -85,7 +85,7 @@ COPY "tools/cc_utils/Cargo.toml" "tools/cc_utils/" COPY "spawn_worker/Cargo.toml" "spawn_worker/" COPY "tests/spawn_from_lib/Cargo.toml" "tests/spawn_from_lib/" RUN find -name "Cargo.toml" | sed -e s#Cargo.toml#src/lib.rs#g | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs -RUN echo trace-obfuscation/benches/replace_trace_tags_benchmark.rs ddtelemetry/benches/ipc.rs tools/src/bin/dedup_headers.rs ddtelemetry/examples/tm-worker-test.rs | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs +RUN echo trace-obfuscation/benches/replace_trace_tags_bench.rs ddtelemetry/benches/ipc.rs tools/src/bin/dedup_headers.rs ddtelemetry/examples/tm-worker-test.rs | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs # cache dependencies RUN cargo fetch --locked diff --git a/trace-obfuscation/Cargo.toml b/trace-obfuscation/Cargo.toml index 9bc82028c..86a072f5b 100644 --- a/trace-obfuscation/Cargo.toml +++ b/trace-obfuscation/Cargo.toml @@ -14,5 +14,5 @@ duplicate = "0.4.1" criterion = "0.4" [[bench]] -name = "replace_trace_tags_benchmark" +name = "replace_trace_tags_bench" harness = false \ No newline at end of file diff --git a/trace-obfuscation/benches/replace_trace_tags_benchmark.rs b/trace-obfuscation/benches/replace_trace_tags_bench.rs similarity index 100% rename from trace-obfuscation/benches/replace_trace_tags_benchmark.rs rename to trace-obfuscation/benches/replace_trace_tags_bench.rs