diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c068e9cf7..57c10f7c4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -46,7 +46,7 @@ jobs: shell: bash run: | # shellcheck disable=SC2046 - cargo clippy --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.71.1 ] && echo -Aunknown-lints -Aclippy::cast_ref_to_mut) + cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.71.1 ] && echo -Aunknown-lints -Aclippy::cast_ref_to_mut) licensecheck: runs-on: ubuntu-latest name: "Presence of licence headers" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9511a49f6..c72d0b98f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,16 +48,16 @@ jobs: - name: "Remove nextest CI report" shell: bash run: rm -rf target/nextest/ci/junit.xml - - name: "[${{ steps.rust-version.outputs.version}}] cargo build --workspace --verbose" - run: cargo build --workspace --verbose - - name: "[${{ steps.rust-version.outputs.version}}] cargo nextest run --workspace --profile ci --verbose -E '!test(tracing_integration_tests::)'" + - name: "[${{ steps.rust-version.outputs.version}}] cargo build --workspace --exclude builder --verbose" + run: cargo build --workspace --exclude builder --verbose + - name: "[${{ steps.rust-version.outputs.version}}] cargo nextest run --workspace --exclude builder --profile ci --verbose -E '!test(tracing_integration_tests::)'" # Run doc tests with cargo test and run tests with nextest and generate junit.xml - run: cargo test --workspace --doc --verbose && cargo nextest run --workspace --profile ci --verbose -E '!test(tracing_integration_tests::)' + run: cargo test --workspace --exclude builder --doc --verbose && cargo nextest run --workspace --exclude builder --profile ci --verbose -E '!test(tracing_integration_tests::)' env: RUST_BACKTRACE: 1 - - name: "[${{ steps.rust-version.outputs.version}}] Tracing integration tests: cargo nextest run --workspace --profile ci --verbose -E 'test(tracing_integration_tests::)'" + - name: "[${{ steps.rust-version.outputs.version}}] Tracing integration tests: cargo nextest run --workspace --exclude builder --profile ci --verbose -E 'test(tracing_integration_tests::)'" if: runner.os == 'Linux' - run: cargo nextest run --workspace --profile ci --verbose -E 'test(tracing_integration_tests::)' + run: cargo nextest run --workspace --exclude builder --profile ci --verbose -E 'test(tracing_integration_tests::)' env: RUST_BACKTRACE: 1 - name: "[${{ steps.rust-version.outputs.version}}] RUSTFLAGS=\"-C prefer-dynamic\" cargo nextest run --package test_spawn_from_lib --features prefer-dynamic -E '!test(tracing_integration_tests::)'" @@ -83,6 +83,12 @@ jobs: include: - platform: "ubuntu-latest" rust_version: "${RUST_VERSION}" + - platform: "ubuntu-latest" + flags: "-C relocation-model=pic" + - platform: "macos-12" + flags: "-C relocation-model=pic" + - platform: "windows-latest" + flags: "-C target-feature=+crt-static" steps: - name: Checkout sources uses: actions/checkout@v4 @@ -94,7 +100,7 @@ jobs: if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then WORKSPACE_PATH=$(cygpath -ua '${{ github.workspace }}') fi - echo "OUTPUT_FOLDER=$WORKSPACE_PATH/artifacts" >> $GITHUB_ENV + echo "LIBDD_OUTPUT_FOLDER=$WORKSPACE_PATH/artifacts" >> $GITHUB_ENV - name: Free Disk Space (Ubuntu only) if: runner.os == 'Linux' && matrix.platform == 'ubuntu-latest' @@ -128,13 +134,10 @@ jobs: - name: "Generate profiling FFI" shell: bash + env: + RUSTFLAGS: "${{ matrix.flags }}" run: | - ./build-profiling-ffi.sh -f data-pipeline-ffi ${OUTPUT_FOLDER}/profiling - - - name: "Generate Telemetry FFI" - shell: bash - run: | - ./build-telemetry-ffi.sh ${OUTPUT_FOLDER}/telemetry + cargo build -p builder --features profiling,telemetry,data-pipeline,symbolizer,crashtracker --release -vv - name: 'Publish libdatadog' uses: actions/upload-artifact@v4 @@ -153,7 +156,7 @@ jobs: set -e mkdir examples/ffi/build_dll cd examples/ffi/build_dll - cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling -DVCRUNTIME_LINK_TYPE=DLL + cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER -DVCRUNTIME_LINK_TYPE=DLL cmake --build . - name: "(Windows) Test building Profiling C bindings - static link vc runtime" @@ -163,7 +166,7 @@ jobs: set -e mkdir examples/ffi/build_static cd examples/ffi/build_static - cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling -DVCRUNTIME_LINK_TYPE=STATIC + cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER -DVCRUNTIME_LINK_TYPE=STATIC cmake --build . - name: "Test building Profiling C bindings" @@ -175,11 +178,11 @@ jobs: cd examples/ffi/build # Add BUILD_SYMBOLIZER variable only for Linux platforms if [[ "${{ matrix.platform }}" == "ubuntu-latest" ]]; then - cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling -DBUILD_SYMBOLIZER=true + cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER -DBUILD_SYMBOLIZER=true cmake --build . ./symbolizer else - cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling + cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER cmake --build . fi @@ -207,9 +210,9 @@ jobs: with: rust_version: cross-centos7 - run: cargo install cross || true - - run: cross build --workspace --target x86_64-unknown-linux-gnu - - run: cross test --workspace --target x86_64-unknown-linux-gnu -- --skip "::single_threaded_tests::" --skip "tracing_integration_tests::" - - run: cross test --workspace --target x86_64-unknown-linux-gnu --exclude bin_tests -- --skip "::tests::" --skip "::api_tests::" --test-threads 1 --skip "tracing_integration_tests::" + - run: cross build --workspace --target x86_64-unknown-linux-gnu --exclude builder + - run: cross test --workspace --target x86_64-unknown-linux-gnu --exclude builder -- --skip "::single_threaded_tests::" --skip "tracing_integration_tests::" + - run: cross test --workspace --target x86_64-unknown-linux-gnu --exclude builder --exclude bin_tests -- --skip "::tests::" --skip "::api_tests::" --test-threads 1 --skip "tracing_integration_tests::" ffi_bake: strategy: diff --git a/Cargo.lock b/Cargo.lock index 6a0ccdd49..194cbb80b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -786,6 +786,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "builder" +version = "12.0.0" +dependencies = [ + "anyhow", + "build_common", + "cmake", + "datadog-profiling-ffi", + "ddcommon-ffi", + "tar", + "tools", +] + [[package]] name = "bumpalo" version = "3.16.0" @@ -2090,6 +2103,18 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys 0.52.0", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -4811,6 +4836,17 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tar" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tarpc" version = "0.31.0" @@ -5961,6 +5997,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys 0.4.13", + "rustix 0.38.32", +] + [[package]] name = "yansi" version = "0.5.1" diff --git a/Cargo.toml b/Cargo.toml index 87089d211..d922d5da4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,39 @@ [workspace] members = [ + "alloc", + "builder", + "crashtracker", + "crashtracker-ffi", + "profiling", + "profiling-ffi", + "profiling-replayer", + "ddcommon", + "ddcommon-ffi", + "ddtelemetry", + "ddtelemetry-ffi", + "tools", + "ipc", + "ipc/macros", + "sidecar", + "sidecar/macros", + "sidecar-ffi", + "tools/cc_utils", + "tools/sidecar_mockgen", + "trace-normalization", + "trace-obfuscation", + "trace-utils", + "spawn_worker", + "tests/spawn_from_lib", + "serverless", + "bin_tests", + "data-pipeline", + "data-pipeline-ffi", + "ddsketch", + "tinybytes", +] + +default-members = [ "alloc", "crashtracker", "crashtracker-ffi", @@ -35,6 +68,7 @@ members = [ "ddsketch", "tinybytes", ] + # https://doc.rust-lang.org/cargo/reference/resolver.html#feature-resolver-version-2 resolver = "2" diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml index 2d75da322..aca6acd97 100644 --- a/LICENSE-3rdparty.yml +++ b/LICENSE-3rdparty.yml @@ -1,4 +1,4 @@ -root_name: datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, datadog-crashtracker-ffi, ddcommon-ffi, build_common, datadog-profiling, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-utils, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-dynamic-configuration, tools, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, datadog-trace-obfuscation, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent, bin_tests +root_name: datadog-alloc, builder, build_common, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-ddsketch, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-utils, ddcommon, ddcommon-ffi, datadog-crashtracker-ffi, datadog-crashtracker, ddtelemetry, datadog-profiling, ddtelemetry-ffi, symbolizer-ffi, tools, datadog-profiling-replayer, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-sidecar, datadog-remote-config, datadog-dynamic-configuration, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, datadog-trace-obfuscation, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent, bin_tests third_party_libraries: - package_name: addr2line package_version: 0.21.0 diff --git a/README.md b/README.md index da91ce269..deb167075 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,26 @@ See [`CONTRIBUTING.md`](CONTRIBUTING.md). Build `libdatadog` as usual with `cargo build`. +#### Builder crate +To generate a release with the builder crate use `cargo build -p builder` this will trigger all the necessary steps to +create the libraries, binaries, headers and package config files needed to use libdatadog in your project. The default +build does not include any capability so in order to add them here is the list of allowed features: +- profiling: includes the profiling ffi calls and headers to the package. +- telemetry: adds the telemetry symbols and headers to the package. +- data-pipeline: includes the data pipeline ffi calls to the library and headers to the package. +- crashtracker: adds crashtracking capabilities to the package. +- symbolizer: adds symbolizer capabilities to the package. + +In order to set an output directory there's the `LIBDD_OUTPUT_FOLDER` environment varibale. Here's an example to create +a package with all the features and save the relese on `/opt/release` folder: +```bash +LIBDD_OUTPUT_FOLDER=/opt/release cargo build -p builder --features profiling,telemetry,data-pipeline,crashtracker,symbolizer +``` + +#### Build scripts +This is the non-prefered way of building a release, it's use is discouraged and it will be soon deprecated in favour of +using the builder crate. + To package a release with the generated ffi header and CMake module, use the `build-profiling-ffi.sh` / `build-telemetry-ffi.sh` helper scripts. Here's an example of using on of these scripts, placing the output inside `/opt/libdatadog`: diff --git a/build-common/src/cbindgen.rs b/build-common/src/cbindgen.rs index 80eb1416c..4f89f61bf 100644 --- a/build-common/src/cbindgen.rs +++ b/build-common/src/cbindgen.rs @@ -7,12 +7,14 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::str; +pub const HEADER_PATH: &str = "include/datadog"; + /// Determines the cargo target directory and deliverables directory. /// /// # Returns /// /// * `(PathBuf, PathBuf)` - The cargo target directory and deliverables directory. -fn determine_paths() -> (PathBuf, PathBuf) { +pub fn determine_paths() -> (PathBuf, PathBuf) { let cargo_target_dir = match env::var_os("CARGO_TARGET_DIR") { Some(dir) => PathBuf::from(dir), None => { @@ -94,7 +96,7 @@ pub fn generate_header(crate_dir: PathBuf, header_name: &str, output_base_dir: P output_base_dir.is_absolute(), "output_base_dir must be an absolute path" ); - let output_path = output_base_dir.join("include/datadog/").join(header_name); + let output_path = output_base_dir.join(HEADER_PATH).join(header_name); if let Some(parent) = output_path.parent() { fs::create_dir_all(parent).expect("Failed to create output directory"); @@ -148,7 +150,7 @@ pub fn copy_and_configure_headers() { /// * `destination` - The destination path fn copy_header(source: &Path, destination: &Path) { let output_path = destination - .join("include/datadog/") + .join(HEADER_PATH) .join(source.file_name().unwrap()); if let Some(parent) = output_path.parent() { diff --git a/builder/Cargo.toml b/builder/Cargo.toml new file mode 100644 index 000000000..dc3b6f1ce --- /dev/null +++ b/builder/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "builder" +build = "build/main.rs" +rust-version.workspace = true +edition.workspace = true +version.workspace = true +license.workspace = true + +[features] +default = [] +crashtracker = ["datadog-profiling-ffi?/crashtracker-receiver", "datadog-profiling-ffi?/crashtracker-collector"] +profiling = ["dep:datadog-profiling-ffi"] +telemetry = ["profiling", "datadog-profiling-ffi?/ddtelemetry-ffi"] +data-pipeline = ["telemetry", "datadog-profiling-ffi?/data-pipeline-ffi"] +symbolizer = ["profiling", "datadog-profiling-ffi?/symbolizer"] + +[build-dependencies] +anyhow = { version = "1.0" } +build_common = { path = "../build-common", features = ["cbindgen"] } +cmake = "0.1.50" +tools = { path = "../tools" } +ddcommon-ffi = { path = "../ddcommon-ffi" } +datadog-profiling-ffi = { path = "../profiling-ffi", optional = true, features = ["cbindgen"] } +tar = "0.4.41" + +[[bin]] +name = "dummy" +test = false +bench = false diff --git a/builder/build/arch/apple.rs b/builder/build/arch/apple.rs new file mode 100644 index 000000000..d86520d3d --- /dev/null +++ b/builder/build/arch/apple.rs @@ -0,0 +1,37 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::process::Command; + +pub const NATIVE_LIBS: &str = + " -framework Security -framework CoreFoundation -liconv -lSystem -lresolv -lc -lm -liconv"; +pub const PROF_DYNAMIC_LIB: &str = "libdatadog_profiling.dylib"; +pub const PROF_STATIC_LIB: &str = "libdatadog_profiling.a"; +pub const PROF_DYNAMIC_LIB_FFI: &str = "libdatadog_profiling_ffi.dylib"; +pub const PROF_STATIC_LIB_FFI: &str = "libdatadog_profiling_ffi.a"; +pub const REMOVE_RPATH: bool = true; +pub const BUILD_CRASHTRACKER: bool = true; + +pub fn fix_rpath(lib_path: &str) { + if REMOVE_RPATH { + let lib_name = lib_path.split("/").last().unwrap(); + + Command::new("install_name_tool") + .arg("-id") + .arg("@rpath/".to_string() + lib_name) + .arg(lib_path) + .spawn() + .expect("Failed to fix rpath"); + } +} + +pub fn strip_libraries(lib_path: &str) { + // objcopy is not available in macos image. Investigate llvm-objcopy + let mut strip = Command::new("strip") + .arg("-S") + .arg(lib_path.to_owned() + "/libdatadog_profiling.dylib") + .spawn() + .expect("Failed to spawn strip"); + + strip.wait().expect("Failed to strip library"); +} diff --git a/builder/build/arch/linux.rs b/builder/build/arch/linux.rs new file mode 100644 index 000000000..a6b23c54b --- /dev/null +++ b/builder/build/arch/linux.rs @@ -0,0 +1,58 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::process::Command; + +pub const NATIVE_LIBS: &str = " -ldl -lrt -lpthread -lc -lm -lrt -lpthread -lutil -ldl -lutil"; +pub const PROF_DYNAMIC_LIB: &str = "libdatadog_profiling.so"; +pub const PROF_STATIC_LIB: &str = "libdatadog_profiling.a"; +pub const PROF_DYNAMIC_LIB_FFI: &str = "libdatadog_profiling_ffi.so"; +pub const PROF_STATIC_LIB_FFI: &str = "libdatadog_profiling_ffi.a"; +pub const REMOVE_RPATH: bool = false; +pub const BUILD_CRASHTRACKER: bool = true; + +pub fn fix_rpath(lib_path: &str) { + if REMOVE_RPATH { + Command::new("patchelf") + .arg("--remove-rpath") + .arg(lib_path) + .spawn() + .expect("failed to remove rpath"); + } +} + +pub fn strip_libraries(lib_path: &str) { + let mut rm_section = Command::new("objcopy") + .arg("--remove-section") + .arg(".llvmbc") + .arg(lib_path.to_owned() + "/libdatadog_profiling.a") + .spawn() + .expect("failed to spawn objcopy"); + + rm_section.wait().expect("Failed to remove llvmbc section"); + + let mut create_debug = Command::new("objcopy") + .arg("--only-keep-debug") + .arg(lib_path.to_owned() + "/libdatadog_profiling.so") + .arg(lib_path.to_owned() + "/libdatadog_profiling.debug") + .spawn() + .expect("Failed to spawn objcopy"); + + create_debug.wait().expect("Failed to extract debug info"); + + let mut strip = Command::new("strip") + .arg("-S") + .arg(lib_path.to_owned() + "/libdatadog_profiling.so") + .spawn() + .expect("Failed to spawn strip"); + + strip.wait().expect("Failed to strip library"); + + let mut debug = Command::new("objcopy") + .arg("--add-gnu-debuglink=".to_string() + lib_path + "/libdatadog_profiling.debug") + .arg(lib_path.to_owned() + "/libdatadog_profiling.so") + .spawn() + .expect("Failed to spawn objcopy"); + + debug.wait().expect("Failed to set debuglink"); +} diff --git a/builder/build/arch/mod.rs b/builder/build/arch/mod.rs new file mode 100644 index 000000000..0b903a9fb --- /dev/null +++ b/builder/build/arch/mod.rs @@ -0,0 +1,28 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +#[cfg(all(target_os = "linux", target_env = "gnu"))] +mod linux; +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +#[cfg(all(target_os = "linux", target_env = "gnu"))] +pub use crate::arch::linux::*; + +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +#[cfg(all(target_os = "linux", target_env = "musl"))] +mod musl; +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +#[cfg(all(target_os = "linux", target_env = "musl"))] +pub use crate::arch::musl::*; + +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +#[cfg(target_os = "macos")] +pub mod apple; +#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] +#[cfg(target_os = "macos")] +pub use crate::arch::apple::*; + +#[cfg(all(target_arch = "x86_64", target_os = "windows"))] +pub mod windows; +#[cfg(all(target_arch = "x86_64", target_os = "windows"))] +pub use crate::arch::windows::*; diff --git a/builder/build/arch/musl.rs b/builder/build/arch/musl.rs new file mode 100644 index 000000000..b2d84b90e --- /dev/null +++ b/builder/build/arch/musl.rs @@ -0,0 +1,58 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::process::Command; + +pub const NATIVE_LIBS: &str = " -lssp_nonshared -lc"; +pub const PROF_DYNAMIC_LIB: &str = "libdatadog_profiling.so"; +pub const PROF_STATIC_LIB: &str = "libdatadog_profiling.a"; +pub const PROF_DYNAMIC_LIB_FFI: &str = "libdatadog_profiling_ffi.so"; +pub const PROF_STATIC_LIB_FFI: &str = "libdatadog_profiling_ffi.a"; +pub const REMOVE_RPATH: bool = false; +pub const BUILD_CRASHTRACKER: bool = true; + +pub fn fix_rpath(lib_path: &str) { + if REMOVE_RPATH { + Command::new("patchelf") + .arg("--remove-rpath") + .arg(lib_path) + .spawn() + .expect("failed to remove rpath"); + } +} + +pub fn strip_libraries(lib_path: &str) { + let mut rm_section = Command::new("objcopy") + .arg("--remove-section") + .arg(".llvmbc") + .arg(lib_path.to_owned() + "/libdatadog_profiling.a") + .spawn() + .expect("failed to spawn objcopy"); + + rm_section.wait().expect("Failed to remove llvmbc section"); + + let mut create_debug = Command::new("objcopy") + .arg("--only-keep-debug") + .arg(lib_path.to_owned() + "/libdatadog_profiling.so") + .arg(lib_path.to_owned() + "/libdatadog_profiling.debug") + .spawn() + .expect("Failed to spawn objcopy"); + + create_debug.wait().expect("Failed to extract debug info"); + + let mut strip = Command::new("strip") + .arg("-s") + .arg(lib_path.to_owned() + "/libdatadog_profiling.so") + .spawn() + .expect("Failed to spawn strip"); + + strip.wait().expect("Failed to strip library"); + + let mut debug = Command::new("objcopy") + .arg("--add-gnu-debuglink=".to_string() + lib_path + "/libdatadog_profiling.debug") + .arg(lib_path.to_owned() + "/libdatadog_profiling.so") + .spawn() + .expect("Failed to spawn objcopy"); + + debug.wait().expect("Failed to set debuglink"); +} diff --git a/builder/build/arch/windows.rs b/builder/build/arch/windows.rs new file mode 100644 index 000000000..79780bf3d --- /dev/null +++ b/builder/build/arch/windows.rs @@ -0,0 +1,13 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +pub const NATIVE_LIBS: &str = ""; +pub const PROF_DYNAMIC_LIB: &str = "datadog_profiling.dll"; +pub const PROF_STATIC_LIB: &str = "datadog_profiling.lib"; +pub const PROF_DYNAMIC_LIB_FFI: &str = "datadog_profiling_ffi.dll"; +pub const PROF_STATIC_LIB_FFI: &str = "datadog_profiling_ffi.lib"; +pub const REMOVE_RPATH: bool = false; +pub const BUILD_CRASHTRACKER: bool = false; + +pub fn fix_rpath(_lib_path: &str) {} +pub fn strip_libraries(_lib_path: &str) {} diff --git a/builder/build/common.rs b/builder/build/common.rs new file mode 100644 index 000000000..b9f8d9750 --- /dev/null +++ b/builder/build/common.rs @@ -0,0 +1,23 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::module::Module; +use anyhow::Result; +use std::fs; +use std::path::PathBuf; +use std::rc::Rc; + +pub struct Common { + pub source_include: Rc, + pub target_include: Rc, +} + +impl Module for Common { + fn install(&self) -> Result<()> { + let target_path: PathBuf = [self.target_include.as_ref(), "common.h"].iter().collect(); + + let origin_path: PathBuf = [self.source_include.as_ref(), "common.h"].iter().collect(); + fs::copy(origin_path, target_path).expect("Failed to copy common.h"); + Ok(()) + } +} diff --git a/builder/build/crashtracker.rs b/builder/build/crashtracker.rs new file mode 100644 index 000000000..6de4dbfc2 --- /dev/null +++ b/builder/build/crashtracker.rs @@ -0,0 +1,48 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::arch; +use crate::module::Module; +use anyhow::Result; +use std::fs; +use std::path::PathBuf; +use std::rc::Rc; + +pub struct CrashTracker { + pub source_include: Rc, + pub target_dir: Rc, + pub target_include: Rc, +} + +impl CrashTracker { + fn add_binaries(&self) -> Result<()> { + let _dst = cmake::Config::new("../crashtracker") + .define("Datadog_ROOT", self.target_dir.as_ref()) + .define("CMAKE_INSTALL_PREFIX", self.target_dir.as_ref()) + .build(); + + Ok(()) + } + + fn add_headers(&self) -> Result<()> { + let origin_path: PathBuf = [self.source_include.as_ref(), "crashtracker.h"] + .iter() + .collect(); + let target_path: PathBuf = [self.target_include.as_ref(), "crashtracker.h"] + .iter() + .collect(); + fs::copy(origin_path, target_path).expect("Failed to copy crashtracker.h"); + + Ok(()) + } +} + +impl Module for CrashTracker { + fn install(&self) -> Result<()> { + self.add_headers()?; + if arch::BUILD_CRASHTRACKER { + self.add_binaries()?; + } + Ok(()) + } +} diff --git a/builder/build/data_pipeline.rs b/builder/build/data_pipeline.rs new file mode 100644 index 000000000..9c3e8ad7b --- /dev/null +++ b/builder/build/data_pipeline.rs @@ -0,0 +1,23 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::module::Module; +use anyhow::Result; +use std::fs; +use std::path::PathBuf; +use std::rc::Rc; + +pub struct DataPipeline { + pub source_include: Rc, + pub target_include: Rc, +} + +impl Module for DataPipeline { + fn install(&self) -> Result<()> { + let origin_path: PathBuf = [&self.source_include, "data-pipeline.h"].iter().collect(); + let target_path: PathBuf = [&self.target_include, "data-pipeline.h"].iter().collect(); + + fs::copy(origin_path, target_path).expect("Failed to copy data pipeline header"); + Ok(()) + } +} diff --git a/builder/build/main.rs b/builder/build/main.rs new file mode 100644 index 000000000..aeb2696fd --- /dev/null +++ b/builder/build/main.rs @@ -0,0 +1,256 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +pub mod arch; +mod common; +#[cfg(feature = "crashtracker")] +mod crashtracker; +mod module; + +#[cfg(feature = "data-pipeline")] +mod data_pipeline; + +#[cfg(feature = "profiling")] +mod profiling; + +#[cfg(feature = "symbolizer")] +mod symbolizer; + +use anyhow::Result; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::rc::Rc; +use std::{env, fs}; + +use build_common::{determine_paths, HEADER_PATH}; +use tools::headers::dedup_headers; + +use crate::common::Common; +#[cfg(feature = "crashtracker")] +use crate::crashtracker::CrashTracker; +#[cfg(feature = "data-pipeline")] +use crate::data_pipeline::DataPipeline; +#[cfg(feature = "profiling")] +use crate::profiling::Profiling; +#[cfg(feature = "symbolizer")] +use crate::symbolizer::Symbolizer; +use module::Module; + +/// [`Builder`] is a structure that holds all the information required to assemble the final +/// workspace artifact. It will manage the different modules which will be in charge of producing +/// the different binaries and source files that will be part of the artifact. The builder will +/// provide the needed information: paths, version, etc, to the different modules so they can +/// install their sub-artifacts on the target folder. +/// The target folder is set through `LIBDD_OUTPUT_FOLDER` environment variable if it is not +/// provided the default target folder will be the builder output directory. +/// +/// # Example +/// +/// ```rust +/// use crate::core::Core; +/// +/// let mut builder = Builder::new(&path, &profile, &version); +/// let core = Box::new(Core { +/// version: builder.version.clone(), +/// }); +/// builder.add_module(core); +/// builder.build()?; +/// builder.pack()?; +/// ``` +struct Builder { + modules: Vec>, + main_header: Rc, + source_inc: Rc, + source_lib: Rc, + target_dir: Rc, + target_lib: Rc, + target_include: Rc, + target_bin: Rc, + target_pkconfig: Rc, + version: Rc, +} + +impl Builder { + /// Creates a new Builder instance + /// + /// # Aguments + /// + /// * `target_dir`: artifact folder. + /// * `profile`: Release configuration: debug or release; + /// * `version`: artifact's version. + /// + /// # Returns + /// + /// A new Builder instance. + fn new(source_dir: &str, target_dir: &str, profile: &str, version: &str) -> Self { + Builder { + modules: Vec::new(), + main_header: "common.h".into(), + source_lib: (source_dir.to_string() + "/" + profile + "/deps").into(), + source_inc: (source_dir.to_string() + "/" + HEADER_PATH).into(), + target_dir: target_dir.into(), + target_lib: (target_dir.to_string() + "/lib").into(), + target_include: (target_dir.to_string() + "/" + HEADER_PATH).into(), + target_bin: (target_dir.to_string() + "/bin").into(), + target_pkconfig: (target_dir.to_string() + "/lib/pkgconfig").into(), + version: version.into(), + } + } + + /// Adds a boxed object which implements Module trait. + fn add_module(&mut self, module: Box) { + self.modules.push(module); + } + + fn create_dir_structure(&self) { + let target = Path::new(self.target_dir.as_ref()); + if fs::metadata(target).is_ok() { + fs::remove_dir_all(Path::new(self.target_dir.as_ref())) + .expect("Failed to clean preexisting target folder"); + } + fs::create_dir_all(Path::new(self.target_dir.as_ref())) + .expect("Failed to create target directory"); + fs::create_dir_all(Path::new(self.target_include.as_ref())) + .expect("Failed to create include directory"); + fs::create_dir_all(Path::new(self.target_lib.as_ref())) + .expect("Failed to create include directory"); + fs::create_dir_all(Path::new(self.target_bin.as_ref())) + .expect("Failed to create include directory"); + fs::create_dir_all(Path::new(self.target_pkconfig.as_ref())) + .expect("Failed to create include directory"); + } + + fn deduplicate_headers(&self) { + let datadog_inc_dir = Path::new(self.source_inc.as_ref()); + + let mut headers: Vec = Vec::new(); + let inc_files = fs::read_dir(datadog_inc_dir).unwrap(); + for file in inc_files.flatten() { + let name = file.file_name().into_string().unwrap(); + if name.ends_with(".h") && !name.eq("common.h") && !name.eq("blazesym.h") { + headers.push(file.path().to_string_lossy().to_string()); + } + } + + let base_header = self.source_inc.to_string() + "/" + self.main_header.as_ref(); + dedup_headers(&base_header, &headers); + } + + // TODO: maybe do this in module's build.rs + fn sanitize_libraries(&self) { + let datadog_lib_dir = Path::new(self.source_lib.as_ref()); + + let libs = fs::read_dir(datadog_lib_dir).unwrap(); + for lib in libs.flatten() { + let name = lib.file_name().into_string().unwrap(); + if name.ends_with(".so") { + arch::fix_rpath(lib.path().to_str().unwrap()); + } + } + } + + fn add_cmake(&self) { + let libs = arch::NATIVE_LIBS.to_owned(); + let output = Command::new("sed") + .arg("s/@Datadog_LIBRARIES@/".to_string() + libs.trim() + "/g") + .arg("../cmake/DatadogConfig.cmake.in") + .output() + .expect("Failed to modify cmake"); + + let cmake_path: PathBuf = [&self.target_dir, "DatadogConfig.cmake"].iter().collect(); + fs::write(cmake_path, output.stdout).expect("writing cmake file failed"); + } + + /// Builds the final artifact by going through all modules and instancing their install method. + /// + /// #Returns + /// + /// Ok(()) if success Err(_) if failure. + fn build(&self) -> Result<()> { + for module in &self.modules { + module.install()?; + } + Ok(()) + } + + /// Generate a tar file with all the intermediate artifacts generated by all the modules.k + /// + /// #Returns + /// + /// Ok(()) if success Err(_) if failure. + fn pack(&self) -> Result<()> { + let tarname = "libdatadog".to_string() + "_v" + &self.version + ".tar"; + let path: PathBuf = [self.target_dir.as_ref(), &tarname].iter().collect(); + let artifact = fs::File::create(path).expect("Failed to create tarfile"); + let mut ar = tar::Builder::new(artifact); + ar.append_dir_all("lib", self.target_lib.as_ref())?; + ar.append_dir("bin", self.target_bin.as_ref())?; + ar.append_dir_all("include/datadog", self.target_include.as_ref())?; + + ar.finish().expect("Failed to write the tarfile"); + Ok(()) + } +} + +fn main() { + // Rerun build script if any of the env vars change. + println!("cargo:rerun-if-env-changed=LIBDD_OUTPUT_FOLDER"); + println!("cargo:rerun-if-env-changed=PROFILE"); + + let (_, source_path) = determine_paths(); + let mut path = env::var("OUT_DIR").unwrap(); + if let Ok(libdd_path) = env::var("LIBDD_OUTPUT_FOLDER") { + path = libdd_path; + } + + let profile = env::var("PROFILE").unwrap(); + let version = env::var("CARGO_PKG_VERSION").unwrap(); + let mut builder = Builder::new(source_path.to_str().unwrap(), &path, &profile, &version); + + builder.create_dir_structure(); + builder.deduplicate_headers(); + builder.sanitize_libraries(); + builder.add_cmake(); + + // add modules based on features + builder.add_module(Box::new(Common { + source_include: builder.source_inc.clone(), + target_include: builder.target_include.clone(), + })); + + #[cfg(feature = "profiling")] + builder.add_module(Box::new(Profiling { + source_include: builder.source_inc.clone(), + source_lib: builder.source_lib.clone(), + target_include: builder.target_include.clone(), + target_lib: builder.target_lib.clone(), + target_pkconfig: builder.target_pkconfig.clone(), + version: builder.version.clone(), + })); + + #[cfg(feature = "data-pipeline")] + builder.add_module(Box::new(DataPipeline { + source_include: builder.source_inc.clone(), + target_include: builder.target_include.clone(), + })); + + #[cfg(feature = "symbolizer")] + builder.add_module(Box::new(Symbolizer { + source_include: builder.source_inc.clone(), + target_include: builder.target_include.clone(), + })); + + #[cfg(feature = "crashtracker")] + builder.add_module(Box::new(CrashTracker { + source_include: builder.source_inc.clone(), + target_include: builder.target_include.clone(), + target_dir: builder.target_dir.clone(), + })); + + // Build artifacts. + let res = builder.build(); + match res { + Ok(_) => builder.pack().unwrap(), + Err(err) => panic!("{}", format!("Building failed: {}", err)), + } +} diff --git a/builder/build/module.rs b/builder/build/module.rs new file mode 100644 index 000000000..c7fd5ff22 --- /dev/null +++ b/builder/build/module.rs @@ -0,0 +1,47 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::Result; + +/// The trait Module is used to handle different modules under the same interface. A [`Module`] is +/// a installation unit used by the builder to install specific artifacts pertaining to certain +/// crate belonging to the workspace. +/// +/// # Examples +/// +/// Assuming there is a crate inside the workspace named core-ffi and this crate produces a library +/// `libcore_ffi.so` and a header file `core-ffi.h`: +/// ``` +/// struct Core { +/// pub source_include: Rc, +/// pub source_lib: Rc, +/// pub target_include: Rc, +/// pub target_lib: Rc, +/// } +/// +/// impl Core { +/// fn add_header(&self) -> Result<()> { +/// let mut origin_path: PathBuf = [&self.source_include, "core.h"].iter().collect(); +/// let mut target_path: PathBuf = [&self.target_include, "core.h"].iter().collect(); +/// fs::copy(&origin_path, &target_path).expect("Failed to copy the header"); +/// Ok(()) +/// } +/// +/// fn add_lib(&self) -> Result<()> { +/// let mut origin_path: PathBuf = [&self.source_lib, "libcore_ffi.so"].iter().collect(); +/// let mut target_path: PathBuf = [&self.target_lib, "libcore_ffi.so"].iter().collect(); +/// fs::copy(&origin_path, &target_path).expect("Failed to copy the library"); +/// } +/// } +/// +/// impl Module for Core { +/// fn install(&self) -> Result<()> { +/// self.add_header()?; +/// self.add_lib()?; +/// Ok(()) +/// } +/// } +/// ``` +pub trait Module { + fn install(&self) -> Result<()>; +} diff --git a/builder/build/profiling.rs b/builder/build/profiling.rs new file mode 100644 index 000000000..f38384a57 --- /dev/null +++ b/builder/build/profiling.rs @@ -0,0 +1,114 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::arch; +use crate::module::Module; +use anyhow::Result; +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::rc::Rc; + +pub struct Profiling { + pub source_include: Rc, + pub source_lib: Rc, + pub target_include: Rc, + pub target_lib: Rc, + pub target_pkconfig: Rc, + pub version: Rc, +} + +impl Profiling { + fn add_headers(&self) -> Result<()> { + // Allowing unused_mut due to the methods mutating the vector are behind a feature flag. + #[allow(unused_mut)] + let mut headers = vec!["profiling.h"]; + #[cfg(feature = "telemetry")] + headers.push("telemetry.h"); + + let mut origin_path: PathBuf = [&self.source_include, "dummy.h"].iter().collect(); + let mut target_path: PathBuf = [&self.target_include, "dummy.h"].iter().collect(); + + for header in headers { + origin_path.set_file_name(header); + target_path.set_file_name(header); + fs::copy(&origin_path, &target_path).expect("Failed to copy the header"); + } + + Ok(()) + } + + fn add_libs(&self) -> Result<()> { + //Create directory + let lib_dir = Path::new(self.target_lib.as_ref()); + fs::create_dir_all(lib_dir).expect("Failed to create pkgconfig directory"); + + let from_dyn: PathBuf = [&self.source_lib, arch::PROF_DYNAMIC_LIB_FFI] + .iter() + .collect(); + let to_dyn: PathBuf = [lib_dir.as_os_str(), OsStr::new(arch::PROF_DYNAMIC_LIB)] + .iter() + .collect(); + + fs::copy(from_dyn, to_dyn).expect("unable to copy dynamic lib"); + + let from_static: PathBuf = [&self.source_lib, arch::PROF_STATIC_LIB_FFI] + .iter() + .collect(); + let to_static: PathBuf = [lib_dir.as_os_str(), OsStr::new(arch::PROF_STATIC_LIB)] + .iter() + .collect(); + fs::copy(from_static, to_static).expect("unable to copy static lib"); + + // Generate debug information + arch::strip_libraries(&self.target_lib); + Ok(()) + } + + fn add_pkg_config(&self) -> Result<()> { + let files: [&str; 3] = [ + "datadog_profiling.pc", + "datadog_profiling_with_rpath.pc", + "datadog_profiling-static.pc", + ]; + + //Create directory + // let pc_dir: PathBuf = [&self, "lib/pkgconfig"].iter().collect(); + let pc_dir = Path::new(self.target_pkconfig.as_ref()); + fs::create_dir_all(pc_dir).expect("Failed to create pkgconfig directory"); + + // Create files + for file in files.iter() { + let file_in = "../profiling-ffi/".to_string() + file + ".in"; + let output = Command::new("sed") + .arg("s/@Datadog_VERSION@/".to_string() + &self.version + "/g") + .arg(&file_in) + .output() + .expect("sed command failed"); + + let pc_file: PathBuf = [pc_dir.as_os_str(), OsStr::new(file)].iter().collect(); + fs::write(&pc_file, &output.stdout).expect("writing pc file failed"); + + if *file == files[2] { + let output = Command::new("sed") + .arg("s/@Datadog_LIBRARIES@/".to_string() + arch::NATIVE_LIBS + "/g") + .arg(&file_in) + .output() + .expect("sed command failed"); + + fs::write(&pc_file, &output.stdout).expect("writing pc file failed"); + } + } + Ok(()) + } +} + +impl Module for Profiling { + fn install(&self) -> Result<()> { + self.add_headers()?; + self.add_libs()?; + self.add_pkg_config()?; + Ok(()) + } +} diff --git a/builder/build/symbolizer.rs b/builder/build/symbolizer.rs new file mode 100644 index 000000000..e176c0ca0 --- /dev/null +++ b/builder/build/symbolizer.rs @@ -0,0 +1,23 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use crate::module::Module; +use anyhow::Result; +use std::fs; +use std::path::PathBuf; +use std::rc::Rc; + +pub struct Symbolizer { + pub source_include: Rc, + pub target_include: Rc, +} + +impl Module for Symbolizer { + fn install(&self) -> Result<()> { + let origin_path: PathBuf = [&self.source_include, "blazesym.h"].iter().collect(); + let target_path: PathBuf = [&self.target_include, "blazesym.h"].iter().collect(); + + fs::copy(origin_path, target_path).expect("Failed to copy data pipeline header"); + Ok(()) + } +} diff --git a/builder/src/bin/dummy.rs b/builder/src/bin/dummy.rs new file mode 100644 index 000000000..15e274398 --- /dev/null +++ b/builder/src/bin/dummy.rs @@ -0,0 +1,4 @@ +// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +pub fn main() {} diff --git a/crashtracker/CMakeLists.txt b/crashtracker/CMakeLists.txt index b18c5f0da..5b61081cc 100644 --- a/crashtracker/CMakeLists.txt +++ b/crashtracker/CMakeLists.txt @@ -4,4 +4,5 @@ project(datadog_profiling_crashtracking_receiver LANGUAGES C CXX) find_package(Datadog REQUIRED) add_executable(libdatadog-crashtracking-receiver libdatadog-crashtracking-receiver.c) +install(TARGETS libdatadog-crashtracking-receiver RUNTIME) target_link_libraries(libdatadog-crashtracking-receiver PRIVATE Datadog::Profiling) diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index cfbf3d999..56c5b125b 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -112,12 +112,14 @@ COPY "data-pipeline/Cargo.toml" "data-pipeline/" COPY "data-pipeline-ffi/Cargo.toml" "data-pipeline-ffi/" COPY "bin_tests/Cargo.toml" "bin_tests/" COPY "tinybytes/Cargo.toml" "tinybytes/" +COPY "builder/Cargo.toml" "builder/" RUN find -name "Cargo.toml" | sed -e s#Cargo.toml#src/lib.rs#g | xargs -n 1 sh -c 'mkdir -p $(dirname $1); touch $1; echo $1' create_stubs RUN echo \ bin_tests/src/bin/crashtracker_bin_test.rs \ bin_tests/src/bin/crashtracker_receiver.rs \ bin_tests/src/bin/crashtracker_unix_socket_receiver.rs \ bin_tests/src/bin/test_the_tests.rs \ + builder/src/bin/dummy.rs \ ddtelemetry/examples/tm-worker-test.rs \ ipc/benches/ipc.rs \ ipc/tarpc/tarpc/examples/compression.rs \ @@ -154,12 +156,12 @@ COPY --from=ffi_build_platform_agnostic_cache /root/.cargo /root/.cargo/ COPY --from=ffi_build_platform_agnostic_cache /build /build WORKDIR /build # cache debug dependency build -RUN cargo build --lib --all +RUN cargo build --lib --workspace --exclude builder # cache release dependency build -RUN cargo build --release --lib --all +RUN cargo build --release --lib --workspace --exclude builder COPY ./ ./ -RUN ./build-profiling-ffi.sh /build/output +RUN LIBDD_OUTPUT_FOLDER=/build/output cargo build -p builder --features profiling,telemetry,data-pipeline,symbolizer,crashtracker --release FROM scratch as ffi_build_output diff --git a/tools/src/bin/dedup_headers.rs b/tools/src/bin/dedup_headers.rs index 97f5640fa..27c5c9a0b 100644 --- a/tools/src/bin/dedup_headers.rs +++ b/tools/src/bin/dedup_headers.rs @@ -1,164 +1,16 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +use tools::headers::dedup_headers; + /// Usage: /// ./dedup_headers ... /// /// All type definitions will be removed from the child_headers, and moved to the base_header /// if they are not already defined in the parent_header -use regex::{Match, Regex, RegexBuilder}; -use std::collections::HashSet; -use std::fs::{File, OpenOptions}; -use std::io::{self, BufReader, BufWriter, Read, Seek, Write}; - -fn collect_definitions(header: &str) -> Vec> { - lazy_static::lazy_static! { - static ref HEADER_TYPE_DECL_RE: Regex = RegexBuilder::new(r"^(/\*\*([^*]|\*+[^*/])*\*+/\n)?(#define [a-zA-Z_0-9]+ [^\n]+|typedef (struct|enum) [a-zA-Z_0-9]+ +(\{.*?\} )?[a-zA-Z_0-9]+;)\n+") - .multi_line(true) - .dot_matches_new_line(true) - .build() - .unwrap(); - } - HEADER_TYPE_DECL_RE.find_iter(header).collect() -} - -fn read(f: &mut BufReader<&File>) -> String { - let mut s = Vec::new(); - f.read_to_end(&mut s).unwrap(); - String::from_utf8(s).unwrap() -} - -fn write_parts(writer: &mut BufWriter<&File>, parts: &[&str]) -> io::Result<()> { - writer.get_ref().set_len(0)?; - writer.rewind()?; - for part in parts { - writer.write_all(part.as_bytes())?; - } - Ok(()) -} - -fn content_without_defs<'a>(content: &'a str, defs: &[Match]) -> Vec<&'a str> { - let mut new_content_parts = Vec::new(); - let mut pos = 0; - for d in defs { - new_content_parts.push(&content[pos..d.start()]); - pos = d.end(); - } - new_content_parts.push(&content[pos..]); - new_content_parts -} - fn main() { - let args: Vec<_> = std::env::args_os().collect(); - - let mut unique_child_defs: Vec = Vec::new(); - let mut present = HashSet::new(); - for child_def in args[2..].iter().flat_map(|p| { - let child_header = OpenOptions::new().read(true).write(true).open(p).unwrap(); - - let child_header_content = read(&mut BufReader::new(&child_header)); - let child_defs = collect_definitions(&child_header_content); - let new_content_parts = content_without_defs(&child_header_content, &child_defs); - - write_parts(&mut BufWriter::new(&child_header), &new_content_parts).unwrap(); - - child_defs - .into_iter() - .map(|m| m.as_str().to_owned()) - .collect::>() - }) { - if present.contains(&child_def) { - continue; - } - unique_child_defs.push(child_def.clone()); - present.insert(child_def); - } - - let base_header = OpenOptions::new() - .read(true) - .write(true) - .open(&args[1]) - .unwrap(); - let base_header_content = read(&mut BufReader::new(&base_header)); - let base_defs = collect_definitions(&base_header_content); - let base_defs_set: HashSet<_> = base_defs.iter().map(Match::as_str).collect(); - - let mut base_new_parts = vec![&base_header_content[..base_defs.last().unwrap().end()]]; - for child_def in &unique_child_defs { - if base_defs_set.contains(child_def.as_str()) { - continue; - } - base_new_parts.push(child_def); - } - base_new_parts.push(&base_header_content[base_defs.last().unwrap().end()..]); - write_parts(&mut BufWriter::new(&base_header), &base_new_parts).unwrap(); -} - -#[test] -fn collect_definitions_comments() { - let header = r"/** - * `QueueId` is a struct that represents a unique identifier for a queue. - * It contains a single field, `inner`, which is a 64-bit unsigned integer. - */ -typedef uint64_t ddog_QueueId; - -/** - * Holds the raw parts of a Rust Vec; it should only be created from Rust, - * never from C. - **/ -typedef struct ddog_Vec_U8 { - const uint8_t *ptr; - uintptr_t len; - uintptr_t capacity; -} ddog_Vec_U8; -"; - let matches = collect_definitions(header); - - assert_eq!(matches.len(), 1); - assert_eq!( - matches[0].as_str(), - r"/** - * Holds the raw parts of a Rust Vec; it should only be created from Rust, - * never from C. - **/ -typedef struct ddog_Vec_U8 { - const uint8_t *ptr; - uintptr_t len; - uintptr_t capacity; -} ddog_Vec_U8; -" - ); - - let header = r"/** foo */ -typedef struct ddog_Vec_U8 { - const uint8_t *ptr; -} ddog_Vec_U8; -"; - let matches = collect_definitions(header); - - assert_eq!(matches.len(), 1); - assert_eq!( - matches[0].as_str(), - r"/** foo */ -typedef struct ddog_Vec_U8 { - const uint8_t *ptr; -} ddog_Vec_U8; -" - ); - - let header = r"/** foo **/ */ -typedef struct ddog_Vec_U8 { - const uint8_t *ptr; -} ddog_Vec_U8; -"; - let matches = collect_definitions(header); - - assert_eq!(matches.len(), 1); - assert_eq!( - matches[0].as_str(), - r"typedef struct ddog_Vec_U8 { - const uint8_t *ptr; -} ddog_Vec_U8; -" - ); + let args: Vec<_> = std::env::args_os() + .flat_map(|arg| arg.into_string()) + .collect(); + dedup_headers(&args[1], &args[2..]); } diff --git a/tools/src/lib.rs b/tools/src/lib.rs index cd34bff2d..480f69c4a 100644 --- a/tools/src/lib.rs +++ b/tools/src/lib.rs @@ -1,2 +1,168 @@ // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 + +pub mod headers { + + use regex::{Match, Regex, RegexBuilder}; + use std::collections::HashSet; + use std::fs::{File, OpenOptions}; + use std::io::{self, BufReader, BufWriter, Read, Seek, Write}; + + fn collect_definitions(header: &str) -> Vec> { + lazy_static::lazy_static! { + static ref HEADER_TYPE_DECL_RE: Regex = RegexBuilder::new(r"^(/\*\*([^*]|\*+[^*/])*\*+/\n)?(#define [a-zA-Z_0-9]+ [^\n]+|typedef (struct|enum) [a-zA-Z_0-9]+ +(\{.*?\} )?[a-zA-Z_0-9]+;)\n+") + .multi_line(true) + .dot_matches_new_line(true) + .build() + .unwrap(); + } + HEADER_TYPE_DECL_RE.find_iter(header).collect() + } + + fn read(f: &mut BufReader<&File>) -> String { + let mut s = Vec::new(); + f.read_to_end(&mut s).unwrap(); + String::from_utf8(s).unwrap() + } + + fn write_parts(writer: &mut BufWriter<&File>, parts: &[&str]) -> io::Result<()> { + writer.get_ref().set_len(0)?; + writer.rewind()?; + for part in parts { + writer.write_all(part.as_bytes())?; + } + Ok(()) + } + + fn content_without_defs<'a>(content: &'a str, defs: &[Match]) -> Vec<&'a str> { + let mut new_content_parts = Vec::new(); + let mut pos = 0; + for d in defs { + new_content_parts.push(&content[pos..d.start()]); + pos = d.end(); + } + new_content_parts.push(&content[pos..]); + new_content_parts + } + + pub fn dedup_headers(base: &str, headers: &[String]) { + let mut unique_child_defs: Vec = Vec::new(); + let mut present = HashSet::new(); + for child_def in headers.iter().flat_map(|p| { + let child_header = OpenOptions::new().read(true).write(true).open(p).unwrap(); + + let child_header_content = read(&mut BufReader::new(&child_header)); + let child_defs = collect_definitions(&child_header_content); + let new_content_parts = content_without_defs(&child_header_content, &child_defs); + + write_parts(&mut BufWriter::new(&child_header), &new_content_parts).unwrap(); + + child_defs + .into_iter() + .map(|m| m.as_str().to_owned()) + .collect::>() + }) { + if present.contains(&child_def) { + continue; + } + unique_child_defs.push(child_def.clone()); + present.insert(child_def); + } + + println!("base: {:?}", base); + let base_header = OpenOptions::new() + .read(true) + .write(true) + .open(base) + .unwrap(); + + let base_header_content = read(&mut BufReader::new(&base_header)); + let base_defs = collect_definitions(&base_header_content); + let base_defs_set: HashSet<_> = base_defs.iter().map(Match::as_str).collect(); + + let mut base_new_parts = vec![&base_header_content[..base_defs.last().unwrap().end()]]; + for child_def in &unique_child_defs { + if base_defs_set.contains(child_def.as_str()) { + continue; + } + base_new_parts.push(child_def); + } + base_new_parts.push(&base_header_content[base_defs.last().unwrap().end()..]); + write_parts(&mut BufWriter::new(&base_header), &base_new_parts).unwrap(); + } + + #[cfg(test)] + mod tests { + use super::*; + + #[ignore] + #[test] + fn collect_definitions_comments() { + let header = r"/** + * `QueueId` is a struct that represents a unique identifier for a queue. + * It contains a single field, `inner`, which is a 64-bit unsigned integer. + */ + typedef uint64_t ddog_QueueId; + + /** + * Holds the raw parts of a Rust Vec; it should only be created from Rust, + * never from C. + **/ + typedef struct ddog_Vec_U8 { + const uint8_t *ptr; + uintptr_t len; + uintptr_t capacity; + } ddog_Vec_U8; + "; + let matches = collect_definitions(header); + + assert_eq!(matches.len(), 1); + assert_eq!( + matches[0].as_str(), + r"/** + * Holds the raw parts of a Rust Vec; it should only be created from Rust, + * never from C. + **/ + typedef struct ddog_Vec_U8 { + const uint8_t *ptr; + uintptr_t len; + uintptr_t capacity; + } ddog_Vec_U8; + " + ); + + let header = r"/** foo */ + typedef struct ddog_Vec_U8 { + const uint8_t *ptr; + } ddog_Vec_U8; + "; + let matches = collect_definitions(header); + + assert_eq!(matches.len(), 1); + assert_eq!( + matches[0].as_str(), + r"/** foo */ + typedef struct ddog_Vec_U8 { + const uint8_t *ptr; + } ddog_Vec_U8; + " + ); + + let header = r"/** foo **/ */ + typedef struct ddog_Vec_U8 { + const uint8_t *ptr; + } ddog_Vec_U8; + "; + let matches = collect_definitions(header); + + assert_eq!(matches.len(), 1); + assert_eq!( + matches[0].as_str(), + r"typedef struct ddog_Vec_U8 { + const uint8_t *ptr; + } ddog_Vec_U8; + " + ); + } + } +} /* Headers */