Add ability to use static builds of libtiledb

This updates the tiledb-sys crate to be able to build a static version
of libtiledb. This allows Rust projects to create statically linked
binaries for distribution.

Cargo.toml

@@ -33,18 +33,23 @@ version = "0.1.0"
 
 [workspace.dependencies]
 anyhow = "1.0"
+armerge = "2"
 arrow = { version = "52.0.0", features = ["prettyprint"] }
 arrow-schema = { version = "52.0.0" }
+bindgen = "0.70"
 cells = { path = "test-utils/cells", version = "0.1.0" }
+cmake = "0.1"
 itertools = "0"
 libtiledb = { path = "tiledb/libtiledb", version = "0.1.0" }
 num-traits = "0.2"
 paste = "1.0"
 proptest = { version = "1.0.0" }
+regex = "1"
 serde = { version = "1", features = ["derive"] }
 serde_json = { version = "1", features = ["float_roundtrip"] }
 signal = { path = "test-utils/signal", version = "0.1.0" }
 strategy-ext = { path = "test-utils/strategy-ext", version = "0.1.0" }
+subprocess = "0.2"
 tempfile = { version = "3" }
 thiserror = { version = "1" }
 tiledb-api = { path = "tiledb/api", version = "0.1.0" }

README.md

@@ -6,23 +6,105 @@ Rust bindings for TileDB. (currently covering ~45% of the TileDB C API).
 Getting Started
 ---
 
-For the time being, these bindings require that libtiledb be installed into
-`/opt/tiledb`. Eventually we'll fix the linking issues to not require this
-but for now it was the easiest to get working with Cargo.
+Using these bindings requires having a copy of libtiledb installed on your
+system where `pkg-config` can find it. There are many different ways this
+can be accomplished. Building from source is shown below for clarity. For other
+installation methods, see the TileDB documentation.
 
-On macOS and Linux, these quick instructions should be enough to get
-`cargo test` running:
+In this example, I'm using `/opt/tiledb` for the installation location. You are
+free to use any path you so desire.
 
 ```sh
-$ cd ~/wherever/you/keep/code
-$ git clone https://github.com/TileDB-Inc/TileDB
-$ cd TileDB
+$ sudo mkdir -p /opt/tiledb
+$ sudo chown your_username /opt/tiledb
+$ git clone https://github.com/TileDB-Inc/TileDB tiledb
+$ cd ~/tiledb/
 $ mkdir build
 $ cd build
-$ ../bootstrap --enable=ccache,serialization,debug --prefix=/opt/tiledb
-$ make -j$(nproc) && make -C tiledb -j$(nproc) install
-$ cd ~/wherever/you/keep/code
-$ git clone https://github.com/TileDB-Inc/tiledb-rs
-$ cd tiled-rs
-$ cargo test
+$ ../bootstrap --enable=s3,serialization,debug --prefix=/opt/tiledb
+$ make -j$(nproc)
+$ make -j$(nproc) install
 ```
+
+Once built, make sure that your `PKG_CONFIG_PATH` includes the path where
+libtiledb was installed:
+
+```sh
+if [[ ":$PKG_CONFIG_PATH:" != *":/opt/tiledb/lib/pkgconfig:"* ]]; then
+  export "PKG_CONFIG_PATH=${PKG_CONFIG_PATH:+${PKG_CONFIG_PATH}:}/opt/tiledb/lib/pkgconfig"
+fi
+```
+
+> [!NOTE]
+> The snippet above likely looks overly complicated for setting an environment
+> variable. The reason for this is that `cargo` will invalidate cached builds
+> if any of the inputs change, including the value of `PKG_CONFIG_PATH`. The
+> shell weirdness above just ensures that we don't add duplicates to the path
+> which can cause unnecessary rebuilds of anything that transitively depends
+> on `tiledb-sys`.
+>
+> An easy way to check if your Rust environment is causing a bunch of
+> unnecessary build churn is by using sub-shells:
+>
+> ```sh
+> $ cargo build
+> $ zsh # or whatever your shell happens to be
+> $ cargo build
+> ```
+>
+> If that second `cargo build` command causes anything at all to be built, you
+> likely have something in your environment that's being mutated on every
+> invocation.
+
+Finally, we can check that everything compiled and can be discovered by
+`pkg-config`:
+
+```sh
+$ pkg-config tiledb --libs
+-L/opt/tiledb/lib -ltiledb
+```
+
+Creating Static Binaries
+---
+
+> [!WARNING]
+> It is highly recommended to use the dynamic linking as described above unless
+> you are specifically working on creating statically linked release builds
+> for distribution. Building statically can take on the order of thirty minutes
+> and these builds are easily invalidated requiring complete rebuilds. This ends
+> up leading to an extremely poor developer experience.
+
+To build libtiledb statically, simply set the `TILEDB_SYS_STATIC` environment
+variable to anything.
+
+```sh
+$ export TILEDB_SYS_STATIC=true
+$ cargo build
+```
+
+> [!NOTE]
+> If you are encountering "weird" failures in CI where the libtiledb build
+> appears to error out for no reason, it is likely that `cmake` is being too
+> aggressive in parallelizing compilation jobs. See the `TILEDB_SYS_JOBS`
+> environment variable below.
+
+Controlling Static Builds
+---
+
+There are a few environment variables you can use to attempt to speed up static
+builds.
+
+* `TILEDB_SYS_JOBS` - If this environment variable is set, it is passed as
+  `-j${TILEDB_SYS_JOBS}` to `cmake`. This can also be useful to limit
+  parallelization in CI where the compiler can end up starving the CI runner
+  of RAM which can result in mysteriously failed builds.
+* `TILEDB_SYS_CCACHE` - You can set this to anything to tell libtiledb to search
+  for either `sccache` or `ccache` while building. Consult documentation for
+  either of those tools if you wish to install them. You'll likely want to use
+  `sccache`.
+* `VCPKG_ROOT` - Setting up an external installation of `vcpkg` will ensure that
+  libtiledb dependencies are cached which speeds up builds tremendously. Consult
+  the `vcpkg` documentation for specifics. Though the gist of it is to clone the
+  vcpkg repository, run the bootstrap script to download binaries, then export
+  the `VCPKG_ROOT` environment variable to point at that directory.
+

tiledb/api/src/query/read/aggregate/mod.rs

@@ -349,7 +349,8 @@ pub trait AggregateQueryBuilder: QueryBuilder {
                 )
             })?;
         } else {
-            let c_field_name: *const i8 =
+            #[allow(clippy::unnecessary_cast)]
+            let c_field_name =
                 handle.field_name.as_ref().unwrap().as_c_str().as_ptr();
             match handle.function {
                 AggregateFunction::Count => unreachable!(

tiledb/api/src/string.rs

@@ -34,15 +34,11 @@ impl TDBString {
     }
 
     pub fn to_string(&self) -> TileDBResult<String> {
-        let mut c_str: *const i8 = out_ptr!();
+        let mut c_str = out_ptr!();
         let mut c_len: usize = 0;
 
         let res = unsafe {
-            ffi::tiledb_string_view(
-                *self.raw,
-                &mut c_str as *mut *const i8,
-                &mut c_len,
-            )
+            ffi::tiledb_string_view(*self.raw, &mut c_str, &mut c_len)
         };
 
         if res == ffi::TILEDB_OK {

tiledb/libtiledb/src/lib.rs

@@ -3,26 +3,19 @@
 //! Provides functions which can be used in a crate's build script
 //! to add `tiledb` as a dynamically linked and loaded library.
 
-/// Emits the cargo build command `cargo:rustc-link-lib=tiledb`.
+/// Configure rpath for crates that depend on tiledb-sys
 ///
-/// This should be called only from the lowest-level crate which depdends
-/// on symbols from `libtiledb`. Crates which depend on the `tiledb-api`
-/// do not need to call this.
-pub fn link() {
-    pkg_config::Config::new()
-        .atleast_version("2.20.0")
-        .probe("tiledb")
-        .expect("Build-time TileDB library missing, version >= 2.4 not found.");
-    println!("cargo:rustc-link-lib=tiledb");
-}
-
-/// Emits cargo build commands to add `libtiledb.so` to a compiled executable's rpath.
-///
-/// This should be called from the build script of any library or executable which
-/// depends on `tiledb-api`, whether directly or indirectly.
+/// Any crate that depends on tiledb-sys should call this function in its
+/// build.rs so that it rustc is correctly configured. Note that for anyone
+/// building static binaries, this is a no-op when tiledb-sys was built
+/// statically.
 pub fn rpath() {
-    let libdir = pkg_config::get_variable("tiledb", "libdir")
-        .expect("Missing tiledb dependency.");
+    let libdir = if let Ok(libdir) = std::env::var("DEP_TILEDB_LIBDIR") {
+        libdir
+    } else {
+        return;
+    };
+
     println!(
         "cargo:rustc-link-arg=-Wl,-rpath,@loader_path,-rpath,$ORIGIN,-rpath,{}",
         libdir

tiledb/queries/Cargo.toml

@@ -10,3 +10,4 @@ tiledb-common = { workspace = true }
 
 [build-dependencies]
 libtiledb = { workspace = true }
+

tiledb/queries/build.rs

@@ -1,3 +1,3 @@
 fn main() {
-    libtiledb::rpath()
+    libtiledb::rpath();
 }

tiledb/sys/Cargo.toml

@@ -2,9 +2,21 @@
 name = "tiledb-sys"
 version = { workspace = true }
 edition = { workspace = true }
+build = "build/main.rs"
+links = "tiledb"
 
 [dependencies]
 tiledb-sys-defs = { workspace = true }
 
 [build-dependencies]
-libtiledb = { workspace = true }
+armerge = { workspace = true }
+bindgen = { workspace = true }
+cmake = { workspace = true }
+pkg-config = { workspace = true }
+regex = { workspace = true }
+subprocess = { workspace = true }
+tiledb-utils = { workspace = true }
+thiserror = { workspace = true }
+
+[features]
+default = []

tiledb/sys/build/command.rs

@@ -0,0 +1,57 @@
+use std::thread;
+use std::time::Duration;
+
+use subprocess as sp;
+
+use crate::error::{Error, Result};
+
+pub fn run(cmd: &[&str], input: Option<&str>) -> Result<()> {
+    // Execute our Git command
+    let stdin = if input.is_some() {
+        sp::Redirection::Pipe
+    } else {
+        sp::Redirection::None
+    };
+
+    let mut git = sp::Popen::create(
+        cmd,
+        sp::PopenConfig {
+            stdin,
+            stdout: sp::Redirection::Pipe,
+            stderr: sp::Redirection::Pipe,
+            ..Default::default()
+        },
+    )
+    .map_err(|e| {
+        Error::Popen(format!("Spawning command: {}", cmd.join(" ")), e)
+    })?;
+
+    // Obtain the output from the standard streams.
+    let (out, err) = git.communicate(input).map_err(|e| {
+        Error::IO(format!("Executing command: {}", cmd.join(" ")), e)
+    })?;
+
+    // Wait for git to finish executing
+    loop {
+        let result = git.poll();
+        if result.is_none() {
+            thread::sleep(Duration::from_secs(1));
+            continue;
+        }
+
+        if !matches!(result, Some(sp::ExitStatus::Exited(0))) {
+            let msg = format!(
+                "Error executing command.\ncommand: {}\ninput: {:?}\nstdout:\n{}\n\nstderr:\n{}",
+                cmd.join(" "),
+                input,
+                out.unwrap_or("".to_string()),
+                err.unwrap_or("".to_string())
+            );
+            return Err(Error::Git(msg));
+        }
+
+        break;
+    }
+
+    Ok(())
+}

tiledb/sys/build/compile.rs

@@ -0,0 +1,52 @@
+use crate::current_os;
+use crate::error::Result;
+use crate::utils;
+
+pub fn libtiledb() -> Result<String> {
+    let build_dir = utils::build_dir();
+    if build_dir.is_dir() {
+        let mut bundled = build_dir.clone();
+        bundled.push("libtiledb_bundled.a");
+        if bundled.is_file() {
+            configure_rustc(&build_dir);
+            return Ok(build_dir.display().to_string());
+        }
+    }
+
+    // N.B., you might think this should be `build_dir()`, but the cmake crate
+    // appends `build` unconditionally so we have to go one directory up.
+    let out_dir = utils::out_dir();
+    let git_dir = utils::git_dir();
+    let mut builder = cmake::Config::new(&git_dir);
+    builder
+        .out_dir(out_dir)
+        .build_target("all")
+        .define("BUILD_SHARED_LIBS", "OFF")
+        .define("TILEDB_WERROR", "OFF")
+        .define("TILEDB_S3", "ON")
+        .define("TILEDB_SERIALIZATION", "ON");
+
+    if std::env::var("TILEDB_SYS_CCACHE").is_ok() {
+        builder.define("TILEDB_CCACHE", "ON");
+    }
+
+    if let Ok(num_jobs) = std::env::var("TILEDB_SYS_JOBS") {
+        builder.build_arg(format!("-j{}", num_jobs));
+    }
+
+    let mut dst = builder.build();
+    dst.push("build");
+
+    current_os::merge_libraries(&dst)?;
+    configure_rustc(&dst);
+    Ok(dst.display().to_string())
+}
+
+fn configure_rustc(out: &std::path::Path) {
+    // Configure linking
+    println!("cargo::rustc-link-search=native={}", out.display());
+    println!("cargo::rustc-link-lib=static=tiledb_bundled");
+
+    // Add any extra OS specific config
+    current_os::configure_rustc(out).expect("Error configuring rustc");
+}