Merge branch 'main' into duncan-harvey/azure-resource-group

.github/CODEOWNERS

@@ -21,3 +21,4 @@ serverless              @Datadog/serverless
 sidecar                 @Datadog/libdatadog-php @Datadog/libdatadog-apm
 sidecar-ffi             @Datadog/libdatadog-php @Datadog/libdatadog-apm
 data-pipeline*/         @Datadog/libdatadog-apm
+ddsketch		@Datadog/libdatadog-apm @Datadog/libdatadog-telemetry

.gitlab-ci.yml

@@ -5,6 +5,9 @@ variables:
     value: "main"
     description: "downstream jobs are triggered on this branch"
 
+include:
+  - local: .gitlab/benchmarks.yml
+
 trigger_internal_build:
   variables:
     LIBDATADOG_COMMIT_BRANCH: $CI_COMMIT_BRANCH
@@ -21,7 +24,7 @@ trigger_internal_build:
 report_failure:
   tags: ["arch:amd64"]
   when: on_failure
-  needs: [trigger_internal_build]
+  needs: [trigger_internal_build, benchmarks]
   # allow_failure: true prevents the job from showing up in github statuses (because it's filtered by gitsync)
   allow_failure: true
   script:

.gitlab/benchmarks.yml

@@ -0,0 +1,34 @@
+variables:
+  BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:libdatadog-benchmarks
+  # The Dockerfile to this image is located at:
+  # https://github.com/DataDog/benchmarking-platform/tree/libdatadog/benchmarks
+
+benchmarks:
+  tags: ["runner:apm-k8s-tweaked-metal"]
+  needs: []
+  image:
+    name: $BASE_CI_IMAGE
+  interruptible: true
+  timeout: 1h
+  script:
+    - export ARTIFACTS_DIR="$(pwd)/reports" && (mkdir "${ARTIFACTS_DIR}" || :)
+    - git clone --branch libdatadog/benchmarks https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform /platform && cd /platform
+    - ./steps/capture-hardware-software-info.sh
+    - ./steps/run-benchmarks.sh
+    - ./steps/analyze-results.sh
+    - "./steps/upload-results-to-s3.sh || :"
+    - "./steps/post-pr-comment.sh || :"
+  artifacts:
+    name: "reports"
+    paths:
+      - reports/
+    expire_in: 3 months
+  variables:
+    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance.
+    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # libdatadog
+    UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built.
+    UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
+    UPSTREAM_REPO_URL: "https://github.com/DataDog/libdatadog" # The HTTP URL to the project's repository.
+
+    KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: libdatadog
+    FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true"

Cargo.toml

@@ -28,7 +28,8 @@ members = [
   "serverless",
   "bin_tests",
   "data-pipeline",
-  "data-pipeline-ffi"
+  "data-pipeline-ffi",
+  "ddsketch",
 ]
 # https://doc.rust-lang.org/cargo/reference/resolver.html#feature-resolver-version-2
 resolver = "2"

benchmark/run_benchmarks_ci.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/
+# SPDX-License-Identifier: Apache-2.0
+
+set -eu
+
+function message() {
+  local message=$1 verbose=${2:-"true"}
+  if [[ "${verbose}" == "true" ]]; then
+    echo "$(date +"%T%:z"): $message"
+  fi
+}
+
+CURRENT_PATH=$(pwd)
+readonly CURRENT_PATH="${CURRENT_PATH%/}"
+readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" &>/dev/null && pwd 2>/dev/null)"
+readonly PROJECT_DIR="${SCRIPT_DIR}/.."
+OUTPUT_DIR="${1:-}"
+
+pushd "${PROJECT_DIR}" > /dev/null
+
+# Run benchmarks
+# TODO https://datadoghq.atlassian.net/browse/APMSP-1228
+# Right now we are only running a single benchmark to test the CI setup, and there is only
+# support for the Criterion SamplingMode::Flat in the new benchmarking framework. This will be
+# worked on going forward.
+message "Running benchmarks"
+cargo bench -p datadog-trace-obfuscation --bench trace_obfuscation -- sql/obfuscate_sql_string
+message "Finished running benchmarks"
+
+# Copy the benchmark results to the output directory
+if [[ -n "${OUTPUT_DIR}" && -d "target" ]]; then
+  # Is this a relative path?
+  if [[ "$OUTPUT_DIR" != /* ]]; then
+    OUTPUT_DIR="${CURRENT_PATH}/${OUTPUT_DIR}"
+  fi
+  message "Copying benchmark results to ${OUTPUT_DIR}"
+  pushd target > /dev/null
+  find criterion -type d -regex '.*/new$' | while read -r dir; do
+    mkdir -p "${OUTPUT_DIR}/${dir}"
+    cp -r "${dir}"/* "${OUTPUT_DIR}/${dir}"
+  done
+  popd > /dev/null
+fi

ddsketch/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "datadog-ddsketch"
+description = "Minimal implementation of Datadog's DDSketch"
+edition.workspace = true
+version.workspace = true
+rust-version.workspace = true
+license.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+prost = "0.11.6"
+
+[build-dependencies]
+prost-build = { version = "0.11.9", optional = true  }
+protoc-bin-vendored = { version = "3.0.0", optional = true }
+
+[features]
+generate-protobuf = ["dep:prost-build", "dep:protoc-bin-vendored"]

ddsketch/build.rs

@@ -0,0 +1,52 @@
+// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+use std::error::Error;
+
+#[cfg(feature = "generate-protobuf")]
+use std::{
+    env,
+    fs::File,
+    io::{Read, Write},
+    path::Path,
+};
+
+#[cfg(feature = "generate-protobuf")]
+const HEADER: &str = "// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+";
+
+fn main() -> Result<(), Box<dyn Error>> {
+    #[cfg(feature = "generate-protobuf")]
+    {
+        // protoc is required to compile proto files. This uses protobuf_src to compile protoc
+        // from the source, setting the env var to tell prost_build where to find it.
+        std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap());
+
+        let mut config = prost_build::Config::new();
+
+        let cur_working_dir = env::var("CARGO_MANIFEST_DIR")?;
+        let output_path = Path::new(&cur_working_dir).join("src");
+
+        config.out_dir(output_path.clone());
+
+        println!("cargo:rerun-if-changed=src/pb/DDSketch.proto");
+        config.compile_protos(&["src/pb/DDSketch.proto"], &["src/"])?;
+
+        prepend_to_file(HEADER.as_bytes(), &output_path.join("pb.rs"))?;
+    }
+
+    Ok(())
+}
+
+#[cfg(feature = "generate-protobuf")]
+fn prepend_to_file(data: &[u8], file_path: &Path) -> Result<(), Box<dyn Error>> {
+    let mut f = File::open(file_path)?;
+    let mut content = data.to_owned();
+    f.read_to_end(&mut content)?;
+
+    let mut f = File::create(file_path)?;
+    f.write_all(content.as_slice())?;
+    Ok(())
+}