feat: Static build libhdfs instead

.github/workflows/ci.yml

@@ -11,98 +11,54 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        hdfs-version: [ "2.10.1", "3.2.3", "3.3.2" ]
         os: [ ubuntu-latest ]
 
     steps:
       - uses: actions/checkout@v3
-
-      - name: Checkout python env
-        uses: actions/setup-python@v3
-        with:
-          python-version: '3.8'
-      - name: Checkout java env
-        uses: actions/setup-java@v3
-        with:
-          distribution: temurin
-          java-version: '11'
-      - name: Setup-hdfs env
-        uses: beyondstorage/setup-hdfs@master
-        with:
-          hdfs-version: ${{ matrix.hdfs-version }}
-
       - uses: ./.github/actions/check
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
-  build:
+  unit:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        hdfs-version: [ "2.10.1", "3.2.3", "3.3.2" ]
-        os: [ ubuntu-latest ]
+        # We should support windows, but so far we can't build hadoop on it.
+        # Visit [Tracking issue of windows support](https://github.com/Xuanwo/hdfs-sys/issues/17) for more details.
+        os: [ ubuntu-latest, macos-latest ]
+        feature: [
+          "hdfs_2_2",
+          "hdfs_2_3",
+          "hdfs_2_4",
+          "hdfs_2_5",
+          "hdfs_2_6",
+          "hdfs_2_7",
+          "hdfs_2_8",
+          "hdfs_2_9",
+          "hdfs_2_10",
+          "hdfs_3_0",
+          "hdfs_3_1",
+          "hdfs_3_2",
+          "hdfs_3_3",
+        ]
 
     steps:
       - uses: actions/checkout@v3
-      - uses: Swatinem/rust-cache@v1
-
-      - name: Checkout python env
-        uses: actions/setup-python@v3
-        with:
-          python-version: '3.8'
-      - name: Checkout java env
-        uses: actions/setup-java@v3
+      - uses: actions/setup-java@v3
         with:
-          distribution: temurin
-          java-version: '11'
-      - name: Setup-hdfs env
-        uses: beyondstorage/setup-hdfs@master
-        with:
-          hdfs-version: ${{ matrix.hdfs-version }}
-
+          distribution: 'temurin'
+          java-version: '8'
       - name: Build
         uses: actions-rs/cargo@v1
         with:
           command: build
-
-  unit:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        hdfs-version: [ "2.10.1", "3.2.3", "3.3.2" ]
-        os: [ ubuntu-latest ]
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Checkout python env
-        uses: actions/setup-python@v3
-        with:
-          python-version: '3.8'
-      - name: Checkout java env
-        uses: actions/setup-java@v3
-        with:
-          distribution: temurin
-          java-version: '11'
-      - name: Setup-hdfs env
-        uses: beyondstorage/setup-hdfs@master
-        with:
-          hdfs-version: ${{ matrix.hdfs-version }}
-
-      - name: Test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: -- --nocapture
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
-
+          args: --features ${{ matrix.feature }}
       - name: Test
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --all-features -- --nocapture
+          args: --features ${{ matrix.feature }} -- --nocapture
         env:
           RUST_LOG: DEBUG
           RUST_BACKTRACE: full
+          LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.JAVA_HOME }}/lib/amd64/server:${{ env.JAVA_HOME }}/jre/lib/server:${{ env.JAVA_HOME }}/jre/lib/amd64/server

Cargo.toml

@@ -9,7 +9,6 @@ name = "hdfs-sys"
 repository = "https://github.com/Xuanwo/hdrs"
 version = "0.1.0"
 links = "hdfs"
-build = "build/main.rs"
 
 [package.metadata.docs.rs]
 all-features = true
@@ -29,3 +28,6 @@ hdfs_3_0 = ["hdfs_2_10"]
 hdfs_3_1 = ["hdfs_3_0"]
 hdfs_3_2 = ["hdfs_3_1"]
 hdfs_3_3 = ["hdfs_3_2"]
+
+[build-dependencies]
+cc = "1.0.73"

README.md

@@ -33,24 +33,23 @@ Enable one feature will also enable all features before it. For example, enable
 
 ## Dependencies
 
-This crate will link to `libhdfs` and `libjvm` dynamically.
+This crate will link to `libjvm` dynamically.
 
 To make this crate works correctly, please make sure the following env set correctly:
 
 - `JAVA_HOME`: `hdfs-sys` will search `${JAVA_HOME}/lib/server` to link `libjvm`.
-- `HADOOP_HOME`: `hdfs-sys` will search `${HADOOP_HOME}/lib/native` to link `libhdfs`.
 
 NOTE: `hdfs-sys` will ignore linking if `DOCS_RS` is set to build docs.
 
 ## Runtime
 
-`libhdfs` uses JNI to call functions provided by jars that provided by hadoop releases. Please make sure `CLASSPATH` is set correctly:
+`hdfs-sys` uses JNI to call functions provided by jars that provided by hadoop releases. Please make sure `CLASSPATH` is set correctly before calling any functions provided by `hdfs-sys`:
 
 ```shell
 export JAVA_HOME=/path/to/java
 export HADOOP_HOME=/path/to/hadoop
-export LD_LIBRARY_PATH=${HADOOP_HOME}/lib/native:${JAVA_HOME}/lib/server
-export CLASSPATH=${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/etc/hadoop/*
+export LD_LIBRARY_PATH=${JAVA_HOME}/lib/server
+export CLASSPATH=$(find $HADOOP_HOME -iname "*.jar" | xargs echo | tr ' ' ':')
 ```
 
 ## Contributing

build.rs

@@ -0,0 +1,159 @@
+use std::env;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Ignore link while building docs.
+    if env::var("DOCS_RS").is_ok() {
+        return Ok(());
+    }
+
+    // Make sure jvm has been linked.
+    let java_home = env::var("JAVA_HOME").expect("JAVA_HOME must be set");
+
+    // I don't know where to find libjvm.so exactly.
+    // The only thing I can do is search everywhere I know.
+    println!("cargo:rustc-link-search=native={java_home}/bin/client");
+    println!("cargo:rustc-link-search=native={java_home}/bin/server");
+    println!("cargo:rustc-link-search=native={java_home}/lib/server");
+    println!("cargo:rustc-link-search=native={java_home}/lib/amd64/server");
+    println!("cargo:rustc-link-search=native={java_home}/jre/lib/server");
+    println!("cargo:rustc-link-search=native={java_home}/jre/lib/amd64/server");
+    println!("cargo:rustc-link-lib=jvm");
+
+    // Static link compiled `libhdfs.a`
+    println!("cargo:rustc-link-lib=static=hdfs");
+
+    let mut builder = cc::Build::new();
+    builder.warnings(false);
+    builder.static_flag(true);
+    builder.static_crt(true);
+
+    // Ignore all warnings from cc as we don't care about code written by Apache Hadoop.
+    builder.flag_if_supported("-w");
+    builder.flag_if_supported("-std=c++17");
+
+    // Inspired by [hadoop-hdfs-native-client/src/CMakeLists.txt](https://github.com/apache/hadoop/blob/trunk/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt)
+    if cfg!(windows) {
+        // Set the optimizer level.
+        builder.flag("-O2");
+        // Set warning level 4.
+        builder.flag("/W4");
+        // Skip "unreferenced formal parameter".
+        builder.flag("/wd4100");
+        // Skip "conditional expression is constant".
+        builder.flag("/wd4127");
+        // Skip deprecated POSIX function warnings.
+        builder.flag("-D_CRT_NONSTDC_NO_DEPRECATE");
+        // Skip CRT non-secure function warnings.  If we can convert usage of
+        // strerror, getenv and ctime to their secure CRT equivalents, then we can
+        // re-enable the CRT non-secure function warnings.
+        builder.flag("-D_CRT_SECURE_NO_WARNINGS");
+        // Omit unneeded headers.
+        builder.flag("-DWIN32_LEAN_AND_MEAN");
+    } else {
+        builder.flag("-fvisibility=hidden");
+        // using old default behavior on GCC >= 10.0
+        builder.flag("-fcommon");
+    }
+
+    // Handle java headers.
+    builder.include(format!("{java_home}/include"));
+    if cfg!(target_os = "linux") {
+        builder.include(format!("{java_home}/include/linux"));
+    }
+    if cfg!(target_os = "macos") {
+        builder.include(format!("{java_home}/include/darwin"));
+    }
+    if cfg!(target_os = "windows") {
+        builder.include(format!("{java_home}/include/win32"));
+    }
+
+    // Choose the latest hdfs version.
+    let mut version = "hdfs_2_2";
+    if cfg!(feature = "hdfs_2_3") {
+        version = "hdfs_2_3"
+    }
+    if cfg!(feature = "hdfs_2_4") {
+        version = "hdfs_2_4"
+    }
+    if cfg!(feature = "hdfs_2_5") {
+        version = "hdfs_2_5"
+    }
+    // Since 2.6, hdfs supports windows.
+    //
+    // We build with src from `hdfs_2_6` but expose earlier ABI like `hdfs_2_2`.
+    // This simple trick makes hdfs-sys works on windows without breaking our ABI promise.
+    if cfg!(target_os = "windows") {
+        version = "hdfs_2_6"
+    }
+    if cfg!(feature = "hdfs_2_6") {
+        version = "hdfs_2_6"
+    }
+    if cfg!(feature = "hdfs_2_7") {
+        version = "hdfs_2_7"
+    }
+    if cfg!(feature = "hdfs_2_8") {
+        version = "hdfs_2_8"
+    }
+    if cfg!(feature = "hdfs_2_9") {
+        version = "hdfs_2_9"
+    }
+    if cfg!(feature = "hdfs_2_10") {
+        version = "hdfs_2_10"
+    }
+    if cfg!(feature = "hdfs_3_0") {
+        version = "hdfs_3_0"
+    }
+    if cfg!(feature = "hdfs_3_1") {
+        version = "hdfs_3_1"
+    }
+    if cfg!(feature = "hdfs_3_2") {
+        version = "hdfs_3_2"
+    }
+    if cfg!(feature = "hdfs_3_3") {
+        version = "hdfs_3_3"
+    }
+
+    builder.include("libhdfs");
+    builder.include(format!("libhdfs/{version}"));
+    builder.file(format!("libhdfs/{version}/exception.c"));
+    builder.file(format!("libhdfs/{version}/jni_helper.c"));
+    builder.file(format!("libhdfs/{version}/hdfs.c"));
+
+    // Since 2.6, we need to include mutexes.
+    if cfg!(feature = "hdfs_2_6") || cfg!(target_os = "windows") {
+        builder.include(format!("libhdfs/{version}/os"));
+
+        if cfg!(target_os = "windows") {
+            builder.include(format!("libhdfs/{version}/os/windows"));
+            builder.file(format!("libhdfs/{version}/os/windows/mutexes.c"));
+            builder.file(format!("libhdfs/{version}/os/windows/thread.c"));
+            builder.file(format!(
+                "libhdfs/{version}/os/windows/thread_local_storage.c"
+            ));
+        } else {
+            builder.include(format!("libhdfs/{version}/os/posix"));
+            builder.file(format!("libhdfs/{version}/os/posix/mutexes.c"));
+            builder.file(format!("libhdfs/{version}/os/posix/thread.c"));
+            builder.file(format!("libhdfs/{version}/os/posix/thread_local_storage.c"));
+        }
+    }
+
+    // From 2.6 to 3.3, we need to include htable (removed in 3.3)
+    if cfg!(feature = "hdfs_2_6") && !cfg!(feature = "hdfs_3_3") {
+        builder.include(format!("libhdfs/{version}/common"));
+        builder.file(format!("libhdfs/{version}/common/htable.c"));
+    }
+
+    // Since 2.8, `hdfs.h` has been moved to `include/hdfs/hdfs.h`
+    if cfg!(feature = "hdfs_2_8") {
+        builder.include(format!("libhdfs/{version}/include"));
+    }
+
+    // Since 3.3, we need to compile `jclasses.c`
+    if cfg!(feature = "hdfs_3_3") {
+        builder.file(format!("libhdfs/{version}/jclasses.c"));
+    }
+
+    builder.compile("hdfs");
+    Ok(())
+}