Add build system infrastructure.

.github/workflows/lint.yml

@@ -44,7 +44,7 @@ jobs:
         run: rustup install ${{ matrix.rust_version }} && rustup default ${{ matrix.rust_version }} && rustup component add clippy
       - name: Run clippy on ${{ matrix.platform }} ${{ matrix.rust_version }} 
         shell: bash
-        run: cargo clippy --all-targets --all-features -- -D warnings "$([ ${{ matrix.rust_version }} = 1.71.1 ] && echo -Aunknown-lints)"
+        run: cargo clippy --workspace --all-targets --all-features -- -D warnings "$([ ${{ matrix.rust_version }} = 1.71.1 ] && echo -Aunknown-lints)"
   licensecheck:
     runs-on: ubuntu-latest
     name: "Presence of licence headers"

.github/workflows/test.yml

@@ -48,16 +48,16 @@ jobs:
       - name: "Remove nextest CI report"
         shell: bash
         run: rm -rf target/nextest/ci/junit.xml
-      - name: "[${{ steps.rust-version.outputs.version}}] cargo build --workspace --verbose"
-        run: cargo build --workspace --verbose
-      - name: "[${{ steps.rust-version.outputs.version}}] cargo nextest run --workspace --profile ci --verbose -E '!test(tracing_integration_tests::)'"
+      - name: "[${{ steps.rust-version.outputs.version}}] cargo build --workspace --exclude builder --verbose"
+        run: cargo build --workspace --exclude builder --verbose
+      - name: "[${{ steps.rust-version.outputs.version}}] cargo nextest run --workspace --exclude builder --profile ci --verbose -E '!test(tracing_integration_tests::)'"
         # Run doc tests with cargo test and run tests with nextest and generate junit.xml
-        run: cargo test --workspace --doc --verbose && cargo nextest run --workspace --profile ci --verbose -E '!test(tracing_integration_tests::)'
+        run: cargo test --workspace --exclude builder --doc --verbose && cargo nextest run --workspace --exclude builder --profile ci --verbose -E '!test(tracing_integration_tests::)'
         env:
           RUST_BACKTRACE: 1
-      - name: "[${{ steps.rust-version.outputs.version}}] Tracing integration tests: cargo nextest run --workspace --profile ci --verbose -E 'test(tracing_integration_tests::)'"
+      - name: "[${{ steps.rust-version.outputs.version}}] Tracing integration tests: cargo nextest run --workspace --exclude builder --profile ci --verbose -E 'test(tracing_integration_tests::)'"
         if: runner.os == 'Linux'
-        run: cargo nextest run --workspace --profile ci --verbose -E 'test(tracing_integration_tests::)'
+        run: cargo nextest run --workspace --exclude builder --profile ci --verbose -E 'test(tracing_integration_tests::)'
         env:
           RUST_BACKTRACE: 1
       - name: "[${{ steps.rust-version.outputs.version}}] RUSTFLAGS=\"-C prefer-dynamic\" cargo nextest run --package test_spawn_from_lib --features prefer-dynamic -E '!test(tracing_integration_tests::)'"
@@ -83,6 +83,12 @@ jobs:
         include:
           - platform: "ubuntu-latest"
             rust_version: "${RUST_VERSION}"
+          - platform: "ubuntu-latest"
+            flags: "-C relocation-model=pic"
+          - platform: "macos-12"
+            flags: "-C relocation-model=pic"
+          - platform: "windows-latest"
+            flags: "-C target-feature=+crt-static"
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -94,7 +100,7 @@ jobs:
           if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then
              WORKSPACE_PATH=$(cygpath -ua '${{ github.workspace }}')
           fi
-          echo "OUTPUT_FOLDER=$WORKSPACE_PATH/artifacts" >> $GITHUB_ENV
+          echo "LIBDD_OUTPUT_FOLDER=$WORKSPACE_PATH/artifacts" >> $GITHUB_ENV
 
       - name: Free Disk Space (Ubuntu only)
         if: runner.os == 'Linux' && matrix.platform == 'ubuntu-latest'
@@ -128,13 +134,10 @@ jobs:
 
       - name: "Generate profiling FFI"
         shell: bash
+        env:
+          RUSTFLAGS: "${{ matrix.flags }}"
         run: |
-           ./build-profiling-ffi.sh -f data-pipeline-ffi ${OUTPUT_FOLDER}/profiling
-
-      - name: "Generate Telemetry FFI"
-        shell: bash
-        run: |
-           ./build-telemetry-ffi.sh ${OUTPUT_FOLDER}/telemetry
+           cargo build -p builder --features profiling,telemetry,data-pipeline,symbolizer,crashtracker --release -vv
 
       - name: 'Publish libdatadog'
         uses: actions/upload-artifact@v4
@@ -153,7 +156,7 @@ jobs:
           set -e
           mkdir examples/ffi/build_dll
           cd examples/ffi/build_dll
-          cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling -DVCRUNTIME_LINK_TYPE=DLL
+          cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER -DVCRUNTIME_LINK_TYPE=DLL
           cmake --build .
 
       - name: "(Windows) Test building Profiling C bindings - static link vc runtime"
@@ -163,7 +166,7 @@ jobs:
           set -e
           mkdir examples/ffi/build_static
           cd examples/ffi/build_static
-          cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling -DVCRUNTIME_LINK_TYPE=STATIC
+          cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER -DVCRUNTIME_LINK_TYPE=STATIC
           cmake --build .
 
       - name: "Test building Profiling C bindings"
@@ -175,11 +178,11 @@ jobs:
           cd examples/ffi/build
           # Add BUILD_SYMBOLIZER variable only for Linux platforms
           if [[ "${{ matrix.platform }}" == "ubuntu-latest" ]]; then
-            cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling -DBUILD_SYMBOLIZER=true
+            cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER -DBUILD_SYMBOLIZER=true
             cmake --build  .
             ./symbolizer
           else
-            cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling
+            cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER
             cmake --build .
           fi
 
@@ -207,9 +210,9 @@ jobs:
         with:
           rust_version: cross-centos7
       - run: cargo install cross || true
-      - run: cross build --workspace --target x86_64-unknown-linux-gnu
-      - run: cross test --workspace --target x86_64-unknown-linux-gnu -- --skip "::single_threaded_tests::" --skip "tracing_integration_tests::"
-      - run: cross test --workspace --target x86_64-unknown-linux-gnu --exclude bin_tests -- --skip "::tests::" --skip "::api_tests::" --test-threads 1 --skip "tracing_integration_tests::"
+      - run: cross build --workspace --target x86_64-unknown-linux-gnu --exclude builder 
+      - run: cross test --workspace --target x86_64-unknown-linux-gnu --exclude builder  -- --skip "::single_threaded_tests::" --skip "tracing_integration_tests::"
+      - run: cross test --workspace --target x86_64-unknown-linux-gnu --exclude builder --exclude bin_tests -- --skip "::tests::" --skip "::api_tests::" --test-threads 1 --skip "tracing_integration_tests::"
 
   ffi_bake:
     strategy:

Cargo.toml

@@ -3,6 +3,39 @@
 
 [workspace]
 members = [
+  "alloc",
+  "builder",
+  "crashtracker",
+  "crashtracker-ffi",
+  "profiling",
+  "profiling-ffi",
+  "profiling-replayer",
+  "ddcommon",
+  "ddcommon-ffi",
+  "ddtelemetry",
+  "ddtelemetry-ffi",
+  "tools",
+  "ipc",
+  "ipc/macros",
+  "sidecar",
+  "sidecar/macros",
+  "sidecar-ffi",
+  "tools/cc_utils",
+  "tools/sidecar_mockgen",
+  "trace-normalization",
+  "trace-obfuscation",
+  "trace-utils",
+  "spawn_worker",
+  "tests/spawn_from_lib",
+  "serverless",
+  "bin_tests",
+  "data-pipeline",
+  "data-pipeline-ffi",
+  "ddsketch",
+  "tinybytes",
+]
+
+default-members = [
   "alloc",
   "crashtracker",
   "crashtracker-ffi",
@@ -35,6 +68,7 @@ members = [
   "ddsketch",
   "tinybytes",
 ]
+
 # https://doc.rust-lang.org/cargo/reference/resolver.html#feature-resolver-version-2
 resolver = "2"
 

LICENSE-3rdparty.yml

@@ -1,4 +1,4 @@
-root_name: datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, datadog-crashtracker-ffi, ddcommon-ffi, build_common, datadog-profiling, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-utils, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-dynamic-configuration, tools, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, datadog-trace-obfuscation, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent, bin_tests
+root_name: datadog-alloc, builder, build_common, datadog-crashtracker-ffi, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, ddcommon-ffi, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-utils, datadog-profiling, ddtelemetry-ffi, symbolizer-ffi, tools, datadog-profiling-replayer, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-sidecar, datadog-remote-config, datadog-dynamic-configuration, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, datadog-trace-obfuscation, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent, bin_tests
 third_party_libraries:
 - package_name: addr2line
   package_version: 0.21.0

build-common/src/cbindgen.rs

@@ -7,12 +7,14 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::str;
 
+pub const HEADER_PATH: &str = "include/datadog";
+
 /// Determines the cargo target directory and deliverables directory.
 ///
 /// # Returns
 ///
 /// * `(PathBuf, PathBuf)` - The cargo target directory and deliverables directory.
-fn determine_paths() -> (PathBuf, PathBuf) {
+pub fn determine_paths() -> (PathBuf, PathBuf) {
     let cargo_target_dir = match env::var_os("CARGO_TARGET_DIR") {
         Some(dir) => PathBuf::from(dir),
         None => {
@@ -94,7 +96,7 @@ pub fn generate_header(crate_dir: PathBuf, header_name: &str, output_base_dir: P
         output_base_dir.is_absolute(),
         "output_base_dir must be an absolute path"
     );
-    let output_path = output_base_dir.join("include/datadog/").join(header_name);
+    let output_path = output_base_dir.join(HEADER_PATH).join(header_name);
 
     if let Some(parent) = output_path.parent() {
         fs::create_dir_all(parent).expect("Failed to create output directory");
@@ -148,7 +150,7 @@ pub fn copy_and_configure_headers() {
 /// * `destination` - The destination path
 fn copy_header(source: &Path, destination: &Path) {
     let output_path = destination
-        .join("include/datadog/")
+        .join(HEADER_PATH)
         .join(source.file_name().unwrap());
 
     if let Some(parent) = output_path.parent() {

builder/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "builder"
+build = "build/main.rs"
+rust-version.workspace = true
+edition.workspace = true
+version.workspace = true
+license.workspace = true
+
+[features]
+default = []
+crashtracker = ["datadog-profiling-ffi?/crashtracker-receiver", "datadog-profiling-ffi?/crashtracker-collector"]
+profiling = ["dep:datadog-profiling-ffi"]
+telemetry = ["profiling", "datadog-profiling-ffi?/ddtelemetry-ffi"]
+data-pipeline = ["telemetry", "datadog-profiling-ffi?/data-pipeline-ffi"]
+symbolizer = ["profiling", "datadog-profiling-ffi?/symbolizer"]
+
+[build-dependencies]
+anyhow = { version = "1.0" }
+build_common = { path = "../build-common", features = ["cbindgen"] }
+cmake = "0.1.50"
+tools = { path = "../tools" }
+ddcommon-ffi = { path = "../ddcommon-ffi" }
+datadog-profiling-ffi = { path = "../profiling-ffi", optional = true, features = ["cbindgen"] }
+tar = "0.4.41"
+
+[[bin]]
+name = "dummy"
+test = false
+bench = false

builder/build/arch/apple.rs

@@ -0,0 +1,37 @@
+// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/
+// SPDX-License-Identifier: Apache-2.0
+
+use std::process::Command;
+
+pub const NATIVE_LIBS: &str =
+    " -framework Security -framework CoreFoundation -liconv -lSystem -lresolv -lc -lm -liconv";
+pub const PROF_DYNAMIC_LIB: &str = "libdatadog_profiling.dylib";
+pub const PROF_STATIC_LIB: &str = "libdatadog_profiling.a";
+pub const PROF_DYNAMIC_LIB_FFI: &str = "libdatadog_profiling_ffi.dylib";
+pub const PROF_STATIC_LIB_FFI: &str = "libdatadog_profiling_ffi.a";
+pub const REMOVE_RPATH: bool = true;
+pub const BUILD_CRASHTRACKER: bool = true;
+
+pub fn fix_rpath(lib_path: &str) {
+    if REMOVE_RPATH {
+        let lib_name = lib_path.split("/").last().unwrap();
+
+        Command::new("install_name_tool")
+            .arg("-id")
+            .arg("@rpath/".to_string() + lib_name)
+            .arg(lib_path)
+            .spawn()
+            .expect("Failed to fix rpath");
+    }
+}
+
+pub fn strip_libraries(lib_path: &str) {
+    // objcopy is not available in macos image. Investigate llvm-objcopy
+    let mut strip = Command::new("strip")
+        .arg("-S")
+        .arg(lib_path.to_owned() + "/libdatadog_profiling.dylib")
+        .spawn()
+        .expect("Failed to spawn strip");
+
+    strip.wait().expect("Failed to strip library");
+}