Skip to content

Commit

Permalink
Add build system infrastructure. (#586)
Browse files Browse the repository at this point in the history
* Add document describing the new build process.
* Add build system infrastructure.
* Set output folder via environment variable.
* Add tar generation.
* Make builder work both in release and debug profiles.
* Avoid deep cloning Strings by using Rc<str>.
* Modify CI files to use the new build system.
* Remove pofiling as default feature.
  • Loading branch information
hoolioh authored Sep 5, 2024
1 parent c035537 commit e899c91
Show file tree
Hide file tree
Showing 25 changed files with 1,070 additions and 182 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ jobs:
shell: bash
run: |
# shellcheck disable=SC2046
cargo clippy --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.71.1 ] && echo -Aunknown-lints -Aclippy::cast_ref_to_mut)
cargo clippy --workspace --all-targets --all-features -- -D warnings $([ ${{ matrix.rust_version }} = 1.71.1 ] && echo -Aunknown-lints -Aclippy::cast_ref_to_mut)
licensecheck:
runs-on: ubuntu-latest
name: "Presence of licence headers"
Expand Down
43 changes: 23 additions & 20 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,16 +48,16 @@ jobs:
- name: "Remove nextest CI report"
shell: bash
run: rm -rf target/nextest/ci/junit.xml
- name: "[${{ steps.rust-version.outputs.version}}] cargo build --workspace --verbose"
run: cargo build --workspace --verbose
- name: "[${{ steps.rust-version.outputs.version}}] cargo nextest run --workspace --profile ci --verbose -E '!test(tracing_integration_tests::)'"
- name: "[${{ steps.rust-version.outputs.version}}] cargo build --workspace --exclude builder --verbose"
run: cargo build --workspace --exclude builder --verbose
- name: "[${{ steps.rust-version.outputs.version}}] cargo nextest run --workspace --exclude builder --profile ci --verbose -E '!test(tracing_integration_tests::)'"
# Run doc tests with cargo test and run tests with nextest and generate junit.xml
run: cargo test --workspace --doc --verbose && cargo nextest run --workspace --profile ci --verbose -E '!test(tracing_integration_tests::)'
run: cargo test --workspace --exclude builder --doc --verbose && cargo nextest run --workspace --exclude builder --profile ci --verbose -E '!test(tracing_integration_tests::)'
env:
RUST_BACKTRACE: 1
- name: "[${{ steps.rust-version.outputs.version}}] Tracing integration tests: cargo nextest run --workspace --profile ci --verbose -E 'test(tracing_integration_tests::)'"
- name: "[${{ steps.rust-version.outputs.version}}] Tracing integration tests: cargo nextest run --workspace --exclude builder --profile ci --verbose -E 'test(tracing_integration_tests::)'"
if: runner.os == 'Linux'
run: cargo nextest run --workspace --profile ci --verbose -E 'test(tracing_integration_tests::)'
run: cargo nextest run --workspace --exclude builder --profile ci --verbose -E 'test(tracing_integration_tests::)'
env:
RUST_BACKTRACE: 1
- name: "[${{ steps.rust-version.outputs.version}}] RUSTFLAGS=\"-C prefer-dynamic\" cargo nextest run --package test_spawn_from_lib --features prefer-dynamic -E '!test(tracing_integration_tests::)'"
Expand All @@ -83,6 +83,12 @@ jobs:
include:
- platform: "ubuntu-latest"
rust_version: "${RUST_VERSION}"
- platform: "ubuntu-latest"
flags: "-C relocation-model=pic"
- platform: "macos-12"
flags: "-C relocation-model=pic"
- platform: "windows-latest"
flags: "-C target-feature=+crt-static"
steps:
- name: Checkout sources
uses: actions/checkout@v4
Expand All @@ -94,7 +100,7 @@ jobs:
if [[ "${{ matrix.platform }}" == "windows-latest" ]]; then
WORKSPACE_PATH=$(cygpath -ua '${{ github.workspace }}')
fi
echo "OUTPUT_FOLDER=$WORKSPACE_PATH/artifacts" >> $GITHUB_ENV
echo "LIBDD_OUTPUT_FOLDER=$WORKSPACE_PATH/artifacts" >> $GITHUB_ENV
- name: Free Disk Space (Ubuntu only)
if: runner.os == 'Linux' && matrix.platform == 'ubuntu-latest'
Expand Down Expand Up @@ -128,13 +134,10 @@ jobs:

- name: "Generate profiling FFI"
shell: bash
env:
RUSTFLAGS: "${{ matrix.flags }}"
run: |
./build-profiling-ffi.sh -f data-pipeline-ffi ${OUTPUT_FOLDER}/profiling
- name: "Generate Telemetry FFI"
shell: bash
run: |
./build-telemetry-ffi.sh ${OUTPUT_FOLDER}/telemetry
cargo build -p builder --features profiling,telemetry,data-pipeline,symbolizer,crashtracker --release -vv
- name: 'Publish libdatadog'
uses: actions/upload-artifact@v4
Expand All @@ -153,7 +156,7 @@ jobs:
set -e
mkdir examples/ffi/build_dll
cd examples/ffi/build_dll
cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling -DVCRUNTIME_LINK_TYPE=DLL
cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER -DVCRUNTIME_LINK_TYPE=DLL
cmake --build .
- name: "(Windows) Test building Profiling C bindings - static link vc runtime"
Expand All @@ -163,7 +166,7 @@ jobs:
set -e
mkdir examples/ffi/build_static
cd examples/ffi/build_static
cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling -DVCRUNTIME_LINK_TYPE=STATIC
cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER -DVCRUNTIME_LINK_TYPE=STATIC
cmake --build .
- name: "Test building Profiling C bindings"
Expand All @@ -175,11 +178,11 @@ jobs:
cd examples/ffi/build
# Add BUILD_SYMBOLIZER variable only for Linux platforms
if [[ "${{ matrix.platform }}" == "ubuntu-latest" ]]; then
cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling -DBUILD_SYMBOLIZER=true
cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER -DBUILD_SYMBOLIZER=true
cmake --build .
./symbolizer
else
cmake -S .. -DDatadog_ROOT=$OUTPUT_FOLDER/profiling
cmake -S .. -DDatadog_ROOT=$LIBDD_OUTPUT_FOLDER
cmake --build .
fi
Expand Down Expand Up @@ -207,9 +210,9 @@ jobs:
with:
rust_version: cross-centos7
- run: cargo install cross || true
- run: cross build --workspace --target x86_64-unknown-linux-gnu
- run: cross test --workspace --target x86_64-unknown-linux-gnu -- --skip "::single_threaded_tests::" --skip "tracing_integration_tests::"
- run: cross test --workspace --target x86_64-unknown-linux-gnu --exclude bin_tests -- --skip "::tests::" --skip "::api_tests::" --test-threads 1 --skip "tracing_integration_tests::"
- run: cross build --workspace --target x86_64-unknown-linux-gnu --exclude builder
- run: cross test --workspace --target x86_64-unknown-linux-gnu --exclude builder -- --skip "::single_threaded_tests::" --skip "tracing_integration_tests::"
- run: cross test --workspace --target x86_64-unknown-linux-gnu --exclude builder --exclude bin_tests -- --skip "::tests::" --skip "::api_tests::" --test-threads 1 --skip "tracing_integration_tests::"

ffi_bake:
strategy:
Expand Down
47 changes: 47 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

34 changes: 34 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,39 @@

[workspace]
members = [
"alloc",
"builder",
"crashtracker",
"crashtracker-ffi",
"profiling",
"profiling-ffi",
"profiling-replayer",
"ddcommon",
"ddcommon-ffi",
"ddtelemetry",
"ddtelemetry-ffi",
"tools",
"ipc",
"ipc/macros",
"sidecar",
"sidecar/macros",
"sidecar-ffi",
"tools/cc_utils",
"tools/sidecar_mockgen",
"trace-normalization",
"trace-obfuscation",
"trace-utils",
"spawn_worker",
"tests/spawn_from_lib",
"serverless",
"bin_tests",
"data-pipeline",
"data-pipeline-ffi",
"ddsketch",
"tinybytes",
]

default-members = [
"alloc",
"crashtracker",
"crashtracker-ffi",
Expand Down Expand Up @@ -35,6 +68,7 @@ members = [
"ddsketch",
"tinybytes",
]

# https://doc.rust-lang.org/cargo/reference/resolver.html#feature-resolver-version-2
resolver = "2"

Expand Down
2 changes: 1 addition & 1 deletion LICENSE-3rdparty.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
root_name: datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, datadog-crashtracker-ffi, ddcommon-ffi, build_common, datadog-profiling, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-utils, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-dynamic-configuration, tools, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, datadog-trace-obfuscation, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent, bin_tests
root_name: datadog-alloc, builder, build_common, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-ddsketch, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-utils, ddcommon, ddcommon-ffi, datadog-crashtracker-ffi, datadog-crashtracker, ddtelemetry, datadog-profiling, ddtelemetry-ffi, symbolizer-ffi, tools, datadog-profiling-replayer, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-sidecar, datadog-remote-config, datadog-dynamic-configuration, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, datadog-trace-obfuscation, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent, bin_tests
third_party_libraries:
- package_name: addr2line
package_version: 0.21.0
Expand Down
20 changes: 20 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,26 @@ See [`CONTRIBUTING.md`](CONTRIBUTING.md).

Build `libdatadog` as usual with `cargo build`.

#### Builder crate
To generate a release with the builder crate use `cargo build -p builder` this will trigger all the necessary steps to
create the libraries, binaries, headers and package config files needed to use libdatadog in your project. The default
build does not include any capability so in order to add them here is the list of allowed features:
- profiling: includes the profiling ffi calls and headers to the package.
- telemetry: adds the telemetry symbols and headers to the package.
- data-pipeline: includes the data pipeline ffi calls to the library and headers to the package.
- crashtracker: adds crashtracking capabilities to the package.
- symbolizer: adds symbolizer capabilities to the package.

In order to set an output directory there's the `LIBDD_OUTPUT_FOLDER` environment varibale. Here's an example to create
a package with all the features and save the relese on `/opt/release` folder:
```bash
LIBDD_OUTPUT_FOLDER=/opt/release cargo build -p builder --features profiling,telemetry,data-pipeline,crashtracker,symbolizer
```

#### Build scripts
This is the non-prefered way of building a release, it's use is discouraged and it will be soon deprecated in favour of
using the builder crate.

To package a release with the generated ffi header and CMake module, use the `build-profiling-ffi.sh` / `build-telemetry-ffi.sh` helper scripts.
Here's an example of using on of these scripts, placing the output inside `/opt/libdatadog`:

Expand Down
8 changes: 5 additions & 3 deletions build-common/src/cbindgen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@ use std::path::{Path, PathBuf};
use std::process::Command;
use std::str;

pub const HEADER_PATH: &str = "include/datadog";

/// Determines the cargo target directory and deliverables directory.
///
/// # Returns
///
/// * `(PathBuf, PathBuf)` - The cargo target directory and deliverables directory.
fn determine_paths() -> (PathBuf, PathBuf) {
pub fn determine_paths() -> (PathBuf, PathBuf) {
let cargo_target_dir = match env::var_os("CARGO_TARGET_DIR") {
Some(dir) => PathBuf::from(dir),
None => {
Expand Down Expand Up @@ -94,7 +96,7 @@ pub fn generate_header(crate_dir: PathBuf, header_name: &str, output_base_dir: P
output_base_dir.is_absolute(),
"output_base_dir must be an absolute path"
);
let output_path = output_base_dir.join("include/datadog/").join(header_name);
let output_path = output_base_dir.join(HEADER_PATH).join(header_name);

if let Some(parent) = output_path.parent() {
fs::create_dir_all(parent).expect("Failed to create output directory");
Expand Down Expand Up @@ -148,7 +150,7 @@ pub fn copy_and_configure_headers() {
/// * `destination` - The destination path
fn copy_header(source: &Path, destination: &Path) {
let output_path = destination
.join("include/datadog/")
.join(HEADER_PATH)
.join(source.file_name().unwrap());

if let Some(parent) = output_path.parent() {
Expand Down
29 changes: 29 additions & 0 deletions builder/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
[package]
name = "builder"
build = "build/main.rs"
rust-version.workspace = true
edition.workspace = true
version.workspace = true
license.workspace = true

[features]
default = []
crashtracker = ["datadog-profiling-ffi?/crashtracker-receiver", "datadog-profiling-ffi?/crashtracker-collector"]
profiling = ["dep:datadog-profiling-ffi"]
telemetry = ["profiling", "datadog-profiling-ffi?/ddtelemetry-ffi"]
data-pipeline = ["telemetry", "datadog-profiling-ffi?/data-pipeline-ffi"]
symbolizer = ["profiling", "datadog-profiling-ffi?/symbolizer"]

[build-dependencies]
anyhow = { version = "1.0" }
build_common = { path = "../build-common", features = ["cbindgen"] }
cmake = "0.1.50"
tools = { path = "../tools" }
ddcommon-ffi = { path = "../ddcommon-ffi" }
datadog-profiling-ffi = { path = "../profiling-ffi", optional = true, features = ["cbindgen"] }
tar = "0.4.41"

[[bin]]
name = "dummy"
test = false
bench = false
37 changes: 37 additions & 0 deletions builder/build/arch/apple.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright 2024-Present Datadog, Inc. https://www.datadoghq.com/
// SPDX-License-Identifier: Apache-2.0

use std::process::Command;

pub const NATIVE_LIBS: &str =
" -framework Security -framework CoreFoundation -liconv -lSystem -lresolv -lc -lm -liconv";
pub const PROF_DYNAMIC_LIB: &str = "libdatadog_profiling.dylib";
pub const PROF_STATIC_LIB: &str = "libdatadog_profiling.a";
pub const PROF_DYNAMIC_LIB_FFI: &str = "libdatadog_profiling_ffi.dylib";
pub const PROF_STATIC_LIB_FFI: &str = "libdatadog_profiling_ffi.a";
pub const REMOVE_RPATH: bool = true;
pub const BUILD_CRASHTRACKER: bool = true;

pub fn fix_rpath(lib_path: &str) {
if REMOVE_RPATH {
let lib_name = lib_path.split("/").last().unwrap();

Command::new("install_name_tool")
.arg("-id")
.arg("@rpath/".to_string() + lib_name)
.arg(lib_path)
.spawn()
.expect("Failed to fix rpath");
}
}

pub fn strip_libraries(lib_path: &str) {
// objcopy is not available in macos image. Investigate llvm-objcopy
let mut strip = Command::new("strip")
.arg("-S")
.arg(lib_path.to_owned() + "/libdatadog_profiling.dylib")
.spawn()
.expect("Failed to spawn strip");

strip.wait().expect("Failed to strip library");
}
Loading

0 comments on commit e899c91

Please sign in to comment.