From ffddc3dc071bde4698bc0653210b075a33c762af Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 14:17:44 +0800 Subject: [PATCH 01/20] feat: Static build libhdfs instead Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 92 +- Cargo.toml | 4 +- README.md | 7 +- build.rs | 109 + build/main.rs | 20 - libhdfs/config.h | 0 libhdfs/hdfs_2_10/common/htable.c | 287 ++ libhdfs/hdfs_2_10/common/htable.h | 161 + libhdfs/hdfs_2_10/exception.c | 239 + libhdfs/hdfs_2_10/exception.h | 157 + libhdfs/hdfs_2_10/hdfs.c | 3591 +++++++++++++++ .../hdfs_2_10/include/hdfs/hdfs.h | 0 libhdfs/hdfs_2_10/jni_helper.c | 595 +++ libhdfs/hdfs_2_10/jni_helper.h | 161 + libhdfs/hdfs_2_10/os/mutexes.h | 55 + libhdfs/hdfs_2_10/os/posix/mutexes.c | 43 + libhdfs/hdfs_2_10/os/posix/platform.h | 34 + libhdfs/hdfs_2_10/os/posix/thread.c | 52 + .../hdfs_2_10/os/posix/thread_local_storage.c | 80 + libhdfs/hdfs_2_10/os/thread.h | 54 + libhdfs/hdfs_2_10/os/thread_local_storage.h | 75 + libhdfs/hdfs_2_10/os/windows/inttypes.h | 28 + libhdfs/hdfs_2_10/os/windows/mutexes.c | 52 + libhdfs/hdfs_2_10/os/windows/platform.h | 86 + libhdfs/hdfs_2_10/os/windows/thread.c | 66 + .../os/windows/thread_local_storage.c | 172 + libhdfs/hdfs_2_10/os/windows/unistd.h | 29 + libhdfs/hdfs_2_2/exception.c | 214 + libhdfs/hdfs_2_2/exception.h | 155 + libhdfs/hdfs_2_2/hdfs.c | 2758 ++++++++++++ .../hdfs_abi_2.2.h => libhdfs/hdfs_2_2/hdfs.h | 0 libhdfs/hdfs_2_2/jni_helper.c | 610 +++ libhdfs/hdfs_2_2/jni_helper.h | 122 + libhdfs/hdfs_2_3/exception.c | 233 + libhdfs/hdfs_2_3/exception.h | 155 + libhdfs/hdfs_2_3/hdfs.c | 3144 ++++++++++++++ .../hdfs_abi_2.3.h => libhdfs/hdfs_2_3/hdfs.h | 0 libhdfs/hdfs_2_3/jni_helper.c | 680 +++ libhdfs/hdfs_2_3/jni_helper.h | 163 + libhdfs/hdfs_2_4/exception.c | 233 + libhdfs/hdfs_2_4/exception.h | 155 + libhdfs/hdfs_2_4/hdfs.c | 3144 ++++++++++++++ .../hdfs_abi_2.4.h => libhdfs/hdfs_2_4/hdfs.h | 0 libhdfs/hdfs_2_4/jni_helper.c | 680 +++ libhdfs/hdfs_2_4/jni_helper.h | 163 + libhdfs/hdfs_2_5/exception.c | 233 + libhdfs/hdfs_2_5/exception.h | 155 + libhdfs/hdfs_2_5/hdfs.c | 3144 ++++++++++++++ .../hdfs_abi_2.5.h => libhdfs/hdfs_2_5/hdfs.h | 0 libhdfs/hdfs_2_5/jni_helper.c | 680 +++ libhdfs/hdfs_2_5/jni_helper.h | 163 + libhdfs/hdfs_2_6/common/htable.c | 271 ++ libhdfs/hdfs_2_6/common/htable.h | 161 + libhdfs/hdfs_2_6/exception.c | 234 + libhdfs/hdfs_2_6/exception.h | 157 + libhdfs/hdfs_2_6/hdfs.c | 3245 ++++++++++++++ .../hdfs_abi_2.6.h => libhdfs/hdfs_2_6/hdfs.h | 0 libhdfs/hdfs_2_6/jni_helper.c | 595 +++ libhdfs/hdfs_2_6/jni_helper.h | 161 + libhdfs/hdfs_2_6/os/mutexes.h | 55 + libhdfs/hdfs_2_6/os/posix/mutexes.c | 43 + libhdfs/hdfs_2_6/os/posix/platform.h | 34 + libhdfs/hdfs_2_6/os/posix/thread.c | 52 + .../hdfs_2_6/os/posix/thread_local_storage.c | 80 + libhdfs/hdfs_2_6/os/thread.h | 54 + libhdfs/hdfs_2_6/os/thread_local_storage.h | 75 + libhdfs/hdfs_2_6/os/windows/inttypes.h | 28 + libhdfs/hdfs_2_6/os/windows/mutexes.c | 52 + libhdfs/hdfs_2_6/os/windows/platform.h | 86 + libhdfs/hdfs_2_6/os/windows/thread.c | 66 + .../os/windows/thread_local_storage.c | 164 + libhdfs/hdfs_2_6/os/windows/unistd.h | 29 + libhdfs/hdfs_2_7/common/htable.c | 287 ++ libhdfs/hdfs_2_7/common/htable.h | 161 + libhdfs/hdfs_2_7/exception.c | 239 + libhdfs/hdfs_2_7/exception.h | 157 + libhdfs/hdfs_2_7/hdfs.c | 3341 ++++++++++++++ .../hdfs_abi_2.7.h => libhdfs/hdfs_2_7/hdfs.h | 0 libhdfs/hdfs_2_7/jni_helper.c | 595 +++ libhdfs/hdfs_2_7/jni_helper.h | 161 + libhdfs/hdfs_2_7/os/mutexes.h | 55 + libhdfs/hdfs_2_7/os/posix/mutexes.c | 43 + libhdfs/hdfs_2_7/os/posix/platform.h | 34 + libhdfs/hdfs_2_7/os/posix/thread.c | 52 + .../hdfs_2_7/os/posix/thread_local_storage.c | 80 + libhdfs/hdfs_2_7/os/thread.h | 54 + libhdfs/hdfs_2_7/os/thread_local_storage.h | 75 + libhdfs/hdfs_2_7/os/windows/inttypes.h | 28 + libhdfs/hdfs_2_7/os/windows/mutexes.c | 52 + libhdfs/hdfs_2_7/os/windows/platform.h | 86 + libhdfs/hdfs_2_7/os/windows/thread.c | 66 + .../os/windows/thread_local_storage.c | 172 + libhdfs/hdfs_2_7/os/windows/unistd.h | 29 + libhdfs/hdfs_2_8/common/htable.c | 287 ++ libhdfs/hdfs_2_8/common/htable.h | 161 + libhdfs/hdfs_2_8/exception.c | 239 + libhdfs/hdfs_2_8/exception.h | 157 + libhdfs/hdfs_2_8/hdfs.c | 3342 ++++++++++++++ .../hdfs_2_8/include/hdfs/hdfs.h | 0 libhdfs/hdfs_2_8/jni_helper.c | 595 +++ libhdfs/hdfs_2_8/jni_helper.h | 161 + libhdfs/hdfs_2_8/os/mutexes.h | 55 + libhdfs/hdfs_2_8/os/posix/mutexes.c | 43 + libhdfs/hdfs_2_8/os/posix/platform.h | 34 + libhdfs/hdfs_2_8/os/posix/thread.c | 52 + .../hdfs_2_8/os/posix/thread_local_storage.c | 80 + libhdfs/hdfs_2_8/os/thread.h | 54 + libhdfs/hdfs_2_8/os/thread_local_storage.h | 75 + libhdfs/hdfs_2_8/os/windows/inttypes.h | 28 + libhdfs/hdfs_2_8/os/windows/mutexes.c | 52 + libhdfs/hdfs_2_8/os/windows/platform.h | 86 + libhdfs/hdfs_2_8/os/windows/thread.c | 66 + .../os/windows/thread_local_storage.c | 172 + libhdfs/hdfs_2_8/os/windows/unistd.h | 29 + libhdfs/hdfs_2_9/common/htable.c | 287 ++ libhdfs/hdfs_2_9/common/htable.h | 161 + libhdfs/hdfs_2_9/exception.c | 239 + libhdfs/hdfs_2_9/exception.h | 157 + libhdfs/hdfs_2_9/hdfs.c | 3438 +++++++++++++++ .../hdfs_2_9/include/hdfs/hdfs.h | 0 libhdfs/hdfs_2_9/jni_helper.c | 595 +++ libhdfs/hdfs_2_9/jni_helper.h | 161 + libhdfs/hdfs_2_9/os/mutexes.h | 55 + libhdfs/hdfs_2_9/os/posix/mutexes.c | 43 + libhdfs/hdfs_2_9/os/posix/platform.h | 34 + libhdfs/hdfs_2_9/os/posix/thread.c | 52 + .../hdfs_2_9/os/posix/thread_local_storage.c | 80 + libhdfs/hdfs_2_9/os/thread.h | 54 + libhdfs/hdfs_2_9/os/thread_local_storage.h | 75 + libhdfs/hdfs_2_9/os/windows/inttypes.h | 28 + libhdfs/hdfs_2_9/os/windows/mutexes.c | 52 + libhdfs/hdfs_2_9/os/windows/platform.h | 86 + libhdfs/hdfs_2_9/os/windows/thread.c | 66 + .../os/windows/thread_local_storage.c | 172 + libhdfs/hdfs_2_9/os/windows/unistd.h | 29 + libhdfs/hdfs_3_0/common/htable.c | 287 ++ libhdfs/hdfs_3_0/common/htable.h | 161 + libhdfs/hdfs_3_0/exception.c | 272 ++ libhdfs/hdfs_3_0/exception.h | 165 + libhdfs/hdfs_3_0/hdfs.c | 3528 +++++++++++++++ .../hdfs_3_0/include/hdfs/hdfs.h | 0 libhdfs/hdfs_3_0/jni_helper.c | 666 +++ libhdfs/hdfs_3_0/jni_helper.h | 196 + libhdfs/hdfs_3_0/os/mutexes.h | 55 + libhdfs/hdfs_3_0/os/posix/mutexes.c | 50 + libhdfs/hdfs_3_0/os/posix/platform.h | 34 + libhdfs/hdfs_3_0/os/posix/thread.c | 52 + .../hdfs_3_0/os/posix/thread_local_storage.c | 106 + libhdfs/hdfs_3_0/os/thread.h | 54 + libhdfs/hdfs_3_0/os/thread_local_storage.h | 100 + libhdfs/hdfs_3_0/os/windows/inttypes.h | 28 + libhdfs/hdfs_3_0/os/windows/mutexes.c | 52 + libhdfs/hdfs_3_0/os/windows/platform.h | 86 + libhdfs/hdfs_3_0/os/windows/thread.c | 66 + .../os/windows/thread_local_storage.c | 206 + libhdfs/hdfs_3_0/os/windows/unistd.h | 29 + libhdfs/hdfs_3_1/common/htable.c | 287 ++ libhdfs/hdfs_3_1/common/htable.h | 161 + libhdfs/hdfs_3_1/exception.c | 272 ++ libhdfs/hdfs_3_1/exception.h | 165 + libhdfs/hdfs_3_1/hdfs.c | 3528 +++++++++++++++ .../hdfs_3_1/include/hdfs/hdfs.h | 0 libhdfs/hdfs_3_1/jni_helper.c | 666 +++ libhdfs/hdfs_3_1/jni_helper.h | 196 + libhdfs/hdfs_3_1/os/mutexes.h | 55 + libhdfs/hdfs_3_1/os/posix/mutexes.c | 50 + libhdfs/hdfs_3_1/os/posix/platform.h | 34 + libhdfs/hdfs_3_1/os/posix/thread.c | 52 + .../hdfs_3_1/os/posix/thread_local_storage.c | 106 + libhdfs/hdfs_3_1/os/thread.h | 54 + libhdfs/hdfs_3_1/os/thread_local_storage.h | 100 + libhdfs/hdfs_3_1/os/windows/inttypes.h | 28 + libhdfs/hdfs_3_1/os/windows/mutexes.c | 52 + libhdfs/hdfs_3_1/os/windows/platform.h | 86 + libhdfs/hdfs_3_1/os/windows/thread.c | 66 + .../os/windows/thread_local_storage.c | 206 + libhdfs/hdfs_3_1/os/windows/unistd.h | 29 + libhdfs/hdfs_3_2/common/htable.c | 287 ++ libhdfs/hdfs_3_2/common/htable.h | 161 + libhdfs/hdfs_3_2/exception.c | 272 ++ libhdfs/hdfs_3_2/exception.h | 165 + libhdfs/hdfs_3_2/hdfs.c | 3528 +++++++++++++++ .../hdfs_3_2/include/hdfs/hdfs.h | 0 libhdfs/hdfs_3_2/jni_helper.c | 666 +++ libhdfs/hdfs_3_2/jni_helper.h | 196 + libhdfs/hdfs_3_2/os/mutexes.h | 55 + libhdfs/hdfs_3_2/os/posix/mutexes.c | 50 + libhdfs/hdfs_3_2/os/posix/platform.h | 34 + libhdfs/hdfs_3_2/os/posix/thread.c | 52 + .../hdfs_3_2/os/posix/thread_local_storage.c | 106 + libhdfs/hdfs_3_2/os/thread.h | 54 + libhdfs/hdfs_3_2/os/thread_local_storage.h | 100 + libhdfs/hdfs_3_2/os/windows/inttypes.h | 28 + libhdfs/hdfs_3_2/os/windows/mutexes.c | 52 + libhdfs/hdfs_3_2/os/windows/platform.h | 86 + libhdfs/hdfs_3_2/os/windows/thread.c | 66 + .../os/windows/thread_local_storage.c | 206 + libhdfs/hdfs_3_2/os/windows/unistd.h | 29 + libhdfs/hdfs_3_3/exception.c | 272 ++ libhdfs/hdfs_3_3/exception.h | 165 + libhdfs/hdfs_3_3/hdfs.c | 3831 +++++++++++++++++ .../hdfs_3_3/include/hdfs/hdfs.h | 0 libhdfs/hdfs_3_3/jclasses.c | 136 + libhdfs/hdfs_3_3/jclasses.h | 112 + libhdfs/hdfs_3_3/jni_helper.c | 964 +++++ libhdfs/hdfs_3_3/jni_helper.h | 221 + libhdfs/hdfs_3_3/os/mutexes.h | 55 + libhdfs/hdfs_3_3/os/posix/mutexes.c | 50 + libhdfs/hdfs_3_3/os/posix/platform.h | 34 + libhdfs/hdfs_3_3/os/posix/thread.c | 52 + .../hdfs_3_3/os/posix/thread_local_storage.c | 193 + libhdfs/hdfs_3_3/os/thread.h | 54 + libhdfs/hdfs_3_3/os/thread_local_storage.h | 100 + libhdfs/hdfs_3_3/os/windows/inttypes.h | 28 + libhdfs/hdfs_3_3/os/windows/mutexes.c | 52 + libhdfs/hdfs_3_3/os/windows/platform.h | 86 + libhdfs/hdfs_3_3/os/windows/thread.c | 66 + .../os/windows/thread_local_storage.c | 265 ++ libhdfs/hdfs_3_3/os/windows/unistd.h | 29 + 219 files changed, 71535 insertions(+), 83 deletions(-) create mode 100644 build.rs delete mode 100644 build/main.rs create mode 100644 libhdfs/config.h create mode 100644 libhdfs/hdfs_2_10/common/htable.c create mode 100644 libhdfs/hdfs_2_10/common/htable.h create mode 100644 libhdfs/hdfs_2_10/exception.c create mode 100644 libhdfs/hdfs_2_10/exception.h create mode 100644 libhdfs/hdfs_2_10/hdfs.c rename docs/include/hdfs_abi_2.10.h => libhdfs/hdfs_2_10/include/hdfs/hdfs.h (100%) create mode 100644 libhdfs/hdfs_2_10/jni_helper.c create mode 100644 libhdfs/hdfs_2_10/jni_helper.h create mode 100644 libhdfs/hdfs_2_10/os/mutexes.h create mode 100644 libhdfs/hdfs_2_10/os/posix/mutexes.c create mode 100644 libhdfs/hdfs_2_10/os/posix/platform.h create mode 100644 libhdfs/hdfs_2_10/os/posix/thread.c create mode 100644 libhdfs/hdfs_2_10/os/posix/thread_local_storage.c create mode 100644 libhdfs/hdfs_2_10/os/thread.h create mode 100644 libhdfs/hdfs_2_10/os/thread_local_storage.h create mode 100644 libhdfs/hdfs_2_10/os/windows/inttypes.h create mode 100644 libhdfs/hdfs_2_10/os/windows/mutexes.c create mode 100644 libhdfs/hdfs_2_10/os/windows/platform.h create mode 100644 libhdfs/hdfs_2_10/os/windows/thread.c create mode 100644 libhdfs/hdfs_2_10/os/windows/thread_local_storage.c create mode 100644 libhdfs/hdfs_2_10/os/windows/unistd.h create mode 100644 libhdfs/hdfs_2_2/exception.c create mode 100644 libhdfs/hdfs_2_2/exception.h create mode 100644 libhdfs/hdfs_2_2/hdfs.c rename docs/include/hdfs_abi_2.2.h => libhdfs/hdfs_2_2/hdfs.h (100%) create mode 100644 libhdfs/hdfs_2_2/jni_helper.c create mode 100644 libhdfs/hdfs_2_2/jni_helper.h create mode 100644 libhdfs/hdfs_2_3/exception.c create mode 100644 libhdfs/hdfs_2_3/exception.h create mode 100644 libhdfs/hdfs_2_3/hdfs.c rename docs/include/hdfs_abi_2.3.h => libhdfs/hdfs_2_3/hdfs.h (100%) create mode 100644 libhdfs/hdfs_2_3/jni_helper.c create mode 100644 libhdfs/hdfs_2_3/jni_helper.h create mode 100644 libhdfs/hdfs_2_4/exception.c create mode 100644 libhdfs/hdfs_2_4/exception.h create mode 100644 libhdfs/hdfs_2_4/hdfs.c rename docs/include/hdfs_abi_2.4.h => libhdfs/hdfs_2_4/hdfs.h (100%) create mode 100644 libhdfs/hdfs_2_4/jni_helper.c create mode 100644 libhdfs/hdfs_2_4/jni_helper.h create mode 100644 libhdfs/hdfs_2_5/exception.c create mode 100644 libhdfs/hdfs_2_5/exception.h create mode 100644 libhdfs/hdfs_2_5/hdfs.c rename docs/include/hdfs_abi_2.5.h => libhdfs/hdfs_2_5/hdfs.h (100%) create mode 100644 libhdfs/hdfs_2_5/jni_helper.c create mode 100644 libhdfs/hdfs_2_5/jni_helper.h create mode 100644 libhdfs/hdfs_2_6/common/htable.c create mode 100644 libhdfs/hdfs_2_6/common/htable.h create mode 100644 libhdfs/hdfs_2_6/exception.c create mode 100644 libhdfs/hdfs_2_6/exception.h create mode 100644 libhdfs/hdfs_2_6/hdfs.c rename docs/include/hdfs_abi_2.6.h => libhdfs/hdfs_2_6/hdfs.h (100%) create mode 100644 libhdfs/hdfs_2_6/jni_helper.c create mode 100644 libhdfs/hdfs_2_6/jni_helper.h create mode 100644 libhdfs/hdfs_2_6/os/mutexes.h create mode 100644 libhdfs/hdfs_2_6/os/posix/mutexes.c create mode 100644 libhdfs/hdfs_2_6/os/posix/platform.h create mode 100644 libhdfs/hdfs_2_6/os/posix/thread.c create mode 100644 libhdfs/hdfs_2_6/os/posix/thread_local_storage.c create mode 100644 libhdfs/hdfs_2_6/os/thread.h create mode 100644 libhdfs/hdfs_2_6/os/thread_local_storage.h create mode 100644 libhdfs/hdfs_2_6/os/windows/inttypes.h create mode 100644 libhdfs/hdfs_2_6/os/windows/mutexes.c create mode 100644 libhdfs/hdfs_2_6/os/windows/platform.h create mode 100644 libhdfs/hdfs_2_6/os/windows/thread.c create mode 100644 libhdfs/hdfs_2_6/os/windows/thread_local_storage.c create mode 100644 libhdfs/hdfs_2_6/os/windows/unistd.h create mode 100644 libhdfs/hdfs_2_7/common/htable.c create mode 100644 libhdfs/hdfs_2_7/common/htable.h create mode 100644 libhdfs/hdfs_2_7/exception.c create mode 100644 libhdfs/hdfs_2_7/exception.h create mode 100644 libhdfs/hdfs_2_7/hdfs.c rename docs/include/hdfs_abi_2.7.h => libhdfs/hdfs_2_7/hdfs.h (100%) create mode 100644 libhdfs/hdfs_2_7/jni_helper.c create mode 100644 libhdfs/hdfs_2_7/jni_helper.h create mode 100644 libhdfs/hdfs_2_7/os/mutexes.h create mode 100644 libhdfs/hdfs_2_7/os/posix/mutexes.c create mode 100644 libhdfs/hdfs_2_7/os/posix/platform.h create mode 100644 libhdfs/hdfs_2_7/os/posix/thread.c create mode 100644 libhdfs/hdfs_2_7/os/posix/thread_local_storage.c create mode 100644 libhdfs/hdfs_2_7/os/thread.h create mode 100644 libhdfs/hdfs_2_7/os/thread_local_storage.h create mode 100644 libhdfs/hdfs_2_7/os/windows/inttypes.h create mode 100644 libhdfs/hdfs_2_7/os/windows/mutexes.c create mode 100644 libhdfs/hdfs_2_7/os/windows/platform.h create mode 100644 libhdfs/hdfs_2_7/os/windows/thread.c create mode 100644 libhdfs/hdfs_2_7/os/windows/thread_local_storage.c create mode 100644 libhdfs/hdfs_2_7/os/windows/unistd.h create mode 100644 libhdfs/hdfs_2_8/common/htable.c create mode 100644 libhdfs/hdfs_2_8/common/htable.h create mode 100644 libhdfs/hdfs_2_8/exception.c create mode 100644 libhdfs/hdfs_2_8/exception.h create mode 100644 libhdfs/hdfs_2_8/hdfs.c rename docs/include/hdfs_abi_2.8.h => libhdfs/hdfs_2_8/include/hdfs/hdfs.h (100%) create mode 100644 libhdfs/hdfs_2_8/jni_helper.c create mode 100644 libhdfs/hdfs_2_8/jni_helper.h create mode 100644 libhdfs/hdfs_2_8/os/mutexes.h create mode 100644 libhdfs/hdfs_2_8/os/posix/mutexes.c create mode 100644 libhdfs/hdfs_2_8/os/posix/platform.h create mode 100644 libhdfs/hdfs_2_8/os/posix/thread.c create mode 100644 libhdfs/hdfs_2_8/os/posix/thread_local_storage.c create mode 100644 libhdfs/hdfs_2_8/os/thread.h create mode 100644 libhdfs/hdfs_2_8/os/thread_local_storage.h create mode 100644 libhdfs/hdfs_2_8/os/windows/inttypes.h create mode 100644 libhdfs/hdfs_2_8/os/windows/mutexes.c create mode 100644 libhdfs/hdfs_2_8/os/windows/platform.h create mode 100644 libhdfs/hdfs_2_8/os/windows/thread.c create mode 100644 libhdfs/hdfs_2_8/os/windows/thread_local_storage.c create mode 100644 libhdfs/hdfs_2_8/os/windows/unistd.h create mode 100644 libhdfs/hdfs_2_9/common/htable.c create mode 100644 libhdfs/hdfs_2_9/common/htable.h create mode 100644 libhdfs/hdfs_2_9/exception.c create mode 100644 libhdfs/hdfs_2_9/exception.h create mode 100644 libhdfs/hdfs_2_9/hdfs.c rename docs/include/hdfs_abi_2.9.h => libhdfs/hdfs_2_9/include/hdfs/hdfs.h (100%) create mode 100644 libhdfs/hdfs_2_9/jni_helper.c create mode 100644 libhdfs/hdfs_2_9/jni_helper.h create mode 100644 libhdfs/hdfs_2_9/os/mutexes.h create mode 100644 libhdfs/hdfs_2_9/os/posix/mutexes.c create mode 100644 libhdfs/hdfs_2_9/os/posix/platform.h create mode 100644 libhdfs/hdfs_2_9/os/posix/thread.c create mode 100644 libhdfs/hdfs_2_9/os/posix/thread_local_storage.c create mode 100644 libhdfs/hdfs_2_9/os/thread.h create mode 100644 libhdfs/hdfs_2_9/os/thread_local_storage.h create mode 100644 libhdfs/hdfs_2_9/os/windows/inttypes.h create mode 100644 libhdfs/hdfs_2_9/os/windows/mutexes.c create mode 100644 libhdfs/hdfs_2_9/os/windows/platform.h create mode 100644 libhdfs/hdfs_2_9/os/windows/thread.c create mode 100644 libhdfs/hdfs_2_9/os/windows/thread_local_storage.c create mode 100644 libhdfs/hdfs_2_9/os/windows/unistd.h create mode 100644 libhdfs/hdfs_3_0/common/htable.c create mode 100644 libhdfs/hdfs_3_0/common/htable.h create mode 100644 libhdfs/hdfs_3_0/exception.c create mode 100644 libhdfs/hdfs_3_0/exception.h create mode 100644 libhdfs/hdfs_3_0/hdfs.c rename docs/include/hdfs_abi_3_0.h => libhdfs/hdfs_3_0/include/hdfs/hdfs.h (100%) create mode 100644 libhdfs/hdfs_3_0/jni_helper.c create mode 100644 libhdfs/hdfs_3_0/jni_helper.h create mode 100644 libhdfs/hdfs_3_0/os/mutexes.h create mode 100644 libhdfs/hdfs_3_0/os/posix/mutexes.c create mode 100644 libhdfs/hdfs_3_0/os/posix/platform.h create mode 100644 libhdfs/hdfs_3_0/os/posix/thread.c create mode 100644 libhdfs/hdfs_3_0/os/posix/thread_local_storage.c create mode 100644 libhdfs/hdfs_3_0/os/thread.h create mode 100644 libhdfs/hdfs_3_0/os/thread_local_storage.h create mode 100644 libhdfs/hdfs_3_0/os/windows/inttypes.h create mode 100644 libhdfs/hdfs_3_0/os/windows/mutexes.c create mode 100644 libhdfs/hdfs_3_0/os/windows/platform.h create mode 100644 libhdfs/hdfs_3_0/os/windows/thread.c create mode 100644 libhdfs/hdfs_3_0/os/windows/thread_local_storage.c create mode 100644 libhdfs/hdfs_3_0/os/windows/unistd.h create mode 100644 libhdfs/hdfs_3_1/common/htable.c create mode 100644 libhdfs/hdfs_3_1/common/htable.h create mode 100644 libhdfs/hdfs_3_1/exception.c create mode 100644 libhdfs/hdfs_3_1/exception.h create mode 100644 libhdfs/hdfs_3_1/hdfs.c rename docs/include/hdfs_abi_3_1.h => libhdfs/hdfs_3_1/include/hdfs/hdfs.h (100%) create mode 100644 libhdfs/hdfs_3_1/jni_helper.c create mode 100644 libhdfs/hdfs_3_1/jni_helper.h create mode 100644 libhdfs/hdfs_3_1/os/mutexes.h create mode 100644 libhdfs/hdfs_3_1/os/posix/mutexes.c create mode 100644 libhdfs/hdfs_3_1/os/posix/platform.h create mode 100644 libhdfs/hdfs_3_1/os/posix/thread.c create mode 100644 libhdfs/hdfs_3_1/os/posix/thread_local_storage.c create mode 100644 libhdfs/hdfs_3_1/os/thread.h create mode 100644 libhdfs/hdfs_3_1/os/thread_local_storage.h create mode 100644 libhdfs/hdfs_3_1/os/windows/inttypes.h create mode 100644 libhdfs/hdfs_3_1/os/windows/mutexes.c create mode 100644 libhdfs/hdfs_3_1/os/windows/platform.h create mode 100644 libhdfs/hdfs_3_1/os/windows/thread.c create mode 100644 libhdfs/hdfs_3_1/os/windows/thread_local_storage.c create mode 100644 libhdfs/hdfs_3_1/os/windows/unistd.h create mode 100644 libhdfs/hdfs_3_2/common/htable.c create mode 100644 libhdfs/hdfs_3_2/common/htable.h create mode 100644 libhdfs/hdfs_3_2/exception.c create mode 100644 libhdfs/hdfs_3_2/exception.h create mode 100644 libhdfs/hdfs_3_2/hdfs.c rename docs/include/hdfs_abi_3_2.h => libhdfs/hdfs_3_2/include/hdfs/hdfs.h (100%) create mode 100644 libhdfs/hdfs_3_2/jni_helper.c create mode 100644 libhdfs/hdfs_3_2/jni_helper.h create mode 100644 libhdfs/hdfs_3_2/os/mutexes.h create mode 100644 libhdfs/hdfs_3_2/os/posix/mutexes.c create mode 100644 libhdfs/hdfs_3_2/os/posix/platform.h create mode 100644 libhdfs/hdfs_3_2/os/posix/thread.c create mode 100644 libhdfs/hdfs_3_2/os/posix/thread_local_storage.c create mode 100644 libhdfs/hdfs_3_2/os/thread.h create mode 100644 libhdfs/hdfs_3_2/os/thread_local_storage.h create mode 100644 libhdfs/hdfs_3_2/os/windows/inttypes.h create mode 100644 libhdfs/hdfs_3_2/os/windows/mutexes.c create mode 100644 libhdfs/hdfs_3_2/os/windows/platform.h create mode 100644 libhdfs/hdfs_3_2/os/windows/thread.c create mode 100644 libhdfs/hdfs_3_2/os/windows/thread_local_storage.c create mode 100644 libhdfs/hdfs_3_2/os/windows/unistd.h create mode 100644 libhdfs/hdfs_3_3/exception.c create mode 100644 libhdfs/hdfs_3_3/exception.h create mode 100644 libhdfs/hdfs_3_3/hdfs.c rename docs/include/hdfs_abi_3_3.h => libhdfs/hdfs_3_3/include/hdfs/hdfs.h (100%) create mode 100644 libhdfs/hdfs_3_3/jclasses.c create mode 100644 libhdfs/hdfs_3_3/jclasses.h create mode 100644 libhdfs/hdfs_3_3/jni_helper.c create mode 100644 libhdfs/hdfs_3_3/jni_helper.h create mode 100644 libhdfs/hdfs_3_3/os/mutexes.h create mode 100644 libhdfs/hdfs_3_3/os/posix/mutexes.c create mode 100644 libhdfs/hdfs_3_3/os/posix/platform.h create mode 100644 libhdfs/hdfs_3_3/os/posix/thread.c create mode 100644 libhdfs/hdfs_3_3/os/posix/thread_local_storage.c create mode 100644 libhdfs/hdfs_3_3/os/thread.h create mode 100644 libhdfs/hdfs_3_3/os/thread_local_storage.h create mode 100644 libhdfs/hdfs_3_3/os/windows/inttypes.h create mode 100644 libhdfs/hdfs_3_3/os/windows/mutexes.c create mode 100644 libhdfs/hdfs_3_3/os/windows/platform.h create mode 100644 libhdfs/hdfs_3_3/os/windows/thread.c create mode 100644 libhdfs/hdfs_3_3/os/windows/thread_local_storage.c create mode 100644 libhdfs/hdfs_3_3/os/windows/unistd.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c404368..30ed795 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,21 +16,6 @@ jobs: steps: - uses: actions/checkout@v3 - - - name: Checkout python env - uses: actions/setup-python@v3 - with: - python-version: '3.8' - - name: Checkout java env - uses: actions/setup-java@v3 - with: - distribution: temurin - java-version: '11' - - name: Setup-hdfs env - uses: beyondstorage/setup-hdfs@master - with: - hdfs-version: ${{ matrix.hdfs-version }} - - uses: ./.github/actions/check with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -39,70 +24,61 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - hdfs-version: [ "2.10.1", "3.2.3", "3.3.2" ] - os: [ ubuntu-latest ] + os: [ ubuntu-latest, macos-latest, windows-latest ] + feature: [ + "hdfs_2_2", + "hdfs_2_3", + "hdfs_2_4", + "hdfs_2_5", + "hdfs_2_6", + "hdfs_2_7", + "hdfs_2_8", + "hdfs_2_9", + "hdfs_2_10", + "hdfs_3_0", + "hdfs_3_1", + "hdfs_3_2", + "hdfs_3_3", + ] steps: - uses: actions/checkout@v3 - uses: Swatinem/rust-cache@v1 - - - name: Checkout python env - uses: actions/setup-python@v3 - with: - python-version: '3.8' - - name: Checkout java env - uses: actions/setup-java@v3 - with: - distribution: temurin - java-version: '11' - - name: Setup-hdfs env - uses: beyondstorage/setup-hdfs@master - with: - hdfs-version: ${{ matrix.hdfs-version }} - - name: Build uses: actions-rs/cargo@v1 with: command: build + args: --features ${{ matrix.feature }} unit: runs-on: ${{ matrix.os }} strategy: matrix: - hdfs-version: [ "2.10.1", "3.2.3", "3.3.2" ] - os: [ ubuntu-latest ] + os: [ ubuntu-latest, macos-latest, windows-latest ] + feature: [ + "hdfs_2_2", + "hdfs_2_3", + "hdfs_2_4", + "hdfs_2_5", + "hdfs_2_6", + "hdfs_2_7", + "hdfs_2_8", + "hdfs_2_9", + "hdfs_2_10", + "hdfs_3_0", + "hdfs_3_1", + "hdfs_3_2", + "hdfs_3_3", + ] steps: - uses: actions/checkout@v3 - - name: Checkout python env - uses: actions/setup-python@v3 - with: - python-version: '3.8' - - name: Checkout java env - uses: actions/setup-java@v3 - with: - distribution: temurin - java-version: '11' - - name: Setup-hdfs env - uses: beyondstorage/setup-hdfs@master - with: - hdfs-version: ${{ matrix.hdfs-version }} - - - name: Test - uses: actions-rs/cargo@v1 - with: - command: test - args: -- --nocapture - env: - RUST_LOG: DEBUG - RUST_BACKTRACE: full - - name: Test uses: actions-rs/cargo@v1 with: command: test - args: --all-features -- --nocapture + args: --features ${{ matrix.feature }} -- --nocapture env: RUST_LOG: DEBUG RUST_BACKTRACE: full diff --git a/Cargo.toml b/Cargo.toml index d32868f..29dcda5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,6 @@ name = "hdfs-sys" repository = "https://github.com/Xuanwo/hdrs" version = "0.1.0" links = "hdfs" -build = "build/main.rs" [package.metadata.docs.rs] all-features = true @@ -29,3 +28,6 @@ hdfs_3_0 = ["hdfs_2_10"] hdfs_3_1 = ["hdfs_3_0"] hdfs_3_2 = ["hdfs_3_1"] hdfs_3_3 = ["hdfs_3_2"] + +[build-dependencies] +cc = "1.0.73" diff --git a/README.md b/README.md index bb5eb89..415738b 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,11 @@ Enable one feature will also enable all features before it. For example, enable ## Dependencies -This crate will link to `libhdfs` and `libjvm` dynamically. +This crate will link to `libjvm` dynamically. To make this crate works correctly, please make sure the following env set correctly: - `JAVA_HOME`: `hdfs-sys` will search `${JAVA_HOME}/lib/server` to link `libjvm`. -- `HADOOP_HOME`: `hdfs-sys` will search `${HADOOP_HOME}/lib/native` to link `libhdfs`. NOTE: `hdfs-sys` will ignore linking if `DOCS_RS` is set to build docs. @@ -49,8 +48,8 @@ NOTE: `hdfs-sys` will ignore linking if `DOCS_RS` is set to build docs. ```shell export JAVA_HOME=/path/to/java export HADOOP_HOME=/path/to/hadoop -export LD_LIBRARY_PATH=${HADOOP_HOME}/lib/native:${JAVA_HOME}/lib/server -export CLASSPATH=${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/etc/hadoop/* +export LD_LIBRARY_PATH=${JAVA_HOME}/lib/server +export CLASSPATH=$(find $HADOOP_HOME -iname "*.jar" | xargs echo | tr ' ' ':') ``` ## Contributing diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..27d9df1 --- /dev/null +++ b/build.rs @@ -0,0 +1,109 @@ +use std::env; + +fn main() -> Result<(), Box> { + // Ignore link while building docs. + if env::var("DOCS_RS").is_ok() { + return Ok(()); + } + + // Make sure jvm has been linked. + let java_home = env::var("JAVA_HOME")?; + println!("cargo:rustc-link-search=native={java_home}/lib/server"); + println!("cargo:rustc-link-lib=jvm"); + + // Static link compiled `libhdfs.a` + println!("cargo:rustc-link-lib=static=hdfs"); + + let mut builder = cc::Build::new(); + builder.warnings(false); + builder.static_flag(true); + builder.static_crt(true); + + { + builder.include(format!("{java_home}/include")); + builder.include(format!("{java_home}/include/linux")); + } + + // Choose the latest version. + let mut version = "hdfs_2_2"; + if cfg!(feature = "hdfs_2_3") { + version = "hdfs_2_3" + } + if cfg!(feature = "hdfs_2_4") { + version = "hdfs_2_4" + } + if cfg!(feature = "hdfs_2_5") { + version = "hdfs_2_5" + } + if cfg!(feature = "hdfs_2_6") { + version = "hdfs_2_6" + } + if cfg!(feature = "hdfs_2_7") { + version = "hdfs_2_7" + } + if cfg!(feature = "hdfs_2_8") { + version = "hdfs_2_8" + } + if cfg!(feature = "hdfs_2_9") { + version = "hdfs_2_9" + } + if cfg!(feature = "hdfs_2_10") { + version = "hdfs_2_10" + } + if cfg!(feature = "hdfs_3_0") { + version = "hdfs_3_0" + } + if cfg!(feature = "hdfs_3_1") { + version = "hdfs_3_1" + } + if cfg!(feature = "hdfs_3_2") { + version = "hdfs_3_2" + } + if cfg!(feature = "hdfs_3_3") { + version = "hdfs_3_3" + } + + builder.include("libhdfs"); + builder.include(format!("libhdfs/{version}")); + builder.file(format!("libhdfs/{version}/exception.c")); + builder.file(format!("libhdfs/{version}/jni_helper.c")); + builder.file(format!("libhdfs/{version}/hdfs.c")); + + // Since 2.6, we need to include mutexes. + if cfg!(feature = "hdfs_2_6") { + builder.include(format!("libhdfs/{version}/os")); + + if cfg!(os = "windows") { + builder.include(format!("libhdfs/{version}/os/windows")); + builder.file(format!("libhdfs/{version}/os/windows/mutexes.c")); + builder.file(format!("libhdfs/{version}/os/windows/thread.c")); + builder.file(format!( + "libhdfs/{version}/os/windows/thread_local_storage.c" + )); + } else { + builder.include(format!("libhdfs/{version}/os/posix")); + builder.file(format!("libhdfs/{version}/os/posix/mutexes.c")); + builder.file(format!("libhdfs/{version}/os/posix/thread.c")); + builder.file(format!("libhdfs/{version}/os/posix/thread_local_storage.c")); + } + } + + // From 2.6 to 3.3, we need to include htable (removed in 3.3) + if cfg!(feature = "hdfs_2_6") && !cfg!(feature = "hdfs_3_3") { + builder.include(format!("libhdfs/{version}/common")); + builder.file(format!("libhdfs/{version}/common/htable.c")); + } + + // Since 2.8, `hdfs.h` has been moved to `include/hdfs/hdfs.h` + if cfg!(feature = "hdfs_2_8") { + builder.include(format!("libhdfs/{version}/include")); + } + + // Since 3.3, we need to compile `jclasses.c` + if cfg!(feature = "hdfs_3_3") { + builder.file(format!("libhdfs/{version}/jclasses.c")); + } + + builder.compile("hdfs"); + Ok(()) +} diff --git a/build/main.rs b/build/main.rs deleted file mode 100644 index 7225f2e..0000000 --- a/build/main.rs +++ /dev/null @@ -1,20 +0,0 @@ -use std::env; - -fn main() -> Result<(), Box> { - // Ignore linking libs if we are building docs. - if env::var("DOCS_RS").is_ok() { - return Ok(()); - } - - // Link to libs: dynamics, static, runtime - - // Make sure jvm has been linked. - let java_home = env::var("JAVA_HOME")?; - println!("cargo:rustc-link-search=native={java_home}/lib/server"); - println!("cargo:rustc-link-lib=jvm"); - - let hadoop_home = env::var("HADOOP_HOME")?; - println!("cargo:rustc-link-search=native={hadoop_home}/lib/native"); - println!("cargo:rustc-link-lib=hdfs"); - Ok(()) -} diff --git a/libhdfs/config.h b/libhdfs/config.h new file mode 100644 index 0000000..e69de29 diff --git a/libhdfs/hdfs_2_10/common/htable.c b/libhdfs/hdfs_2_10/common/htable.c new file mode 100644 index 0000000..50c89ea --- /dev/null +++ b/libhdfs/hdfs_2_10/common/htable.c @@ -0,0 +1,287 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/htable.h" + +#include +#include +#include +#include +#include + +struct htable_pair { + void *key; + void *val; +}; + +/** + * A hash table which uses linear probing. + */ +struct htable { + uint32_t capacity; + uint32_t used; + htable_hash_fn_t hash_fun; + htable_eq_fn_t eq_fun; + struct htable_pair *elem; +}; + +/** + * An internal function for inserting a value into the hash table. + * + * Note: this function assumes that you have made enough space in the table. + * + * @param nelem The new element to insert. + * @param capacity The capacity of the hash table. + * @param hash_fun The hash function to use. + * @param key The key to insert. + * @param val The value to insert. + */ +static void htable_insert_internal(struct htable_pair *nelem, + uint32_t capacity, htable_hash_fn_t hash_fun, void *key, + void *val) +{ + uint32_t i; + + i = hash_fun(key, capacity); + while (1) { + if (!nelem[i].key) { + nelem[i].key = key; + nelem[i].val = val; + return; + } + i++; + if (i == capacity) { + i = 0; + } + } +} + +static int htable_realloc(struct htable *htable, uint32_t new_capacity) +{ + struct htable_pair *nelem; + uint32_t i, old_capacity = htable->capacity; + htable_hash_fn_t hash_fun = htable->hash_fun; + + nelem = calloc(new_capacity, sizeof(struct htable_pair)); + if (!nelem) { + return ENOMEM; + } + for (i = 0; i < old_capacity; i++) { + struct htable_pair *pair = htable->elem + i; + if (pair->key) { + htable_insert_internal(nelem, new_capacity, hash_fun, + pair->key, pair->val); + } + } + free(htable->elem); + htable->elem = nelem; + htable->capacity = new_capacity; + return 0; +} + +static uint32_t round_up_to_power_of_2(uint32_t i) +{ + if (i == 0) { + return 1; + } + i--; + i |= i >> 1; + i |= i >> 2; + i |= i >> 4; + i |= i >> 8; + i |= i >> 16; + i++; + return i; +} + +struct htable *htable_alloc(uint32_t size, + htable_hash_fn_t hash_fun, htable_eq_fn_t eq_fun) +{ + struct htable *htable; + + htable = calloc(1, sizeof(*htable)); + if (!htable) { + return NULL; + } + size = round_up_to_power_of_2(size); + if (size < HTABLE_MIN_SIZE) { + size = HTABLE_MIN_SIZE; + } + htable->hash_fun = hash_fun; + htable->eq_fun = eq_fun; + htable->used = 0; + if (htable_realloc(htable, size)) { + free(htable); + return NULL; + } + return htable; +} + +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx) +{ + uint32_t i; + + for (i = 0; i != htable->capacity; ++i) { + struct htable_pair *elem = htable->elem + i; + if (elem->key) { + fun(ctx, elem->key, elem->val); + } + } +} + +void htable_free(struct htable *htable) +{ + if (htable) { + free(htable->elem); + free(htable); + } +} + +int htable_put(struct htable *htable, void *key, void *val) +{ + int ret; + uint32_t nused; + + // NULL is not a valid key value. + // This helps us implement htable_get_internal efficiently, since we know + // that we can stop when we encounter the first NULL key. + if (!key) { + return EINVAL; + } + // NULL is not a valid value. Otherwise the results of htable_get would + // be confusing (does a NULL return mean entry not found, or that the + // entry was found and was NULL?) + if (!val) { + return EINVAL; + } + // Re-hash if we have used more than half of the hash table + nused = htable->used + 1; + if (nused >= (htable->capacity / 2)) { + ret = htable_realloc(htable, htable->capacity * 2); + if (ret) + return ret; + } + htable_insert_internal(htable->elem, htable->capacity, + htable->hash_fun, key, val); + htable->used++; + return 0; +} + +static int htable_get_internal(const struct htable *htable, + const void *key, uint32_t *out) +{ + uint32_t start_idx, idx; + + start_idx = htable->hash_fun(key, htable->capacity); + idx = start_idx; + while (1) { + struct htable_pair *pair = htable->elem + idx; + if (!pair->key) { + // We always maintain the invariant that the entries corresponding + // to a given key are stored in a contiguous block, not separated + // by any NULLs. So if we encounter a NULL, our search is over. + return ENOENT; + } else if (htable->eq_fun(pair->key, key)) { + *out = idx; + return 0; + } + idx++; + if (idx == htable->capacity) { + idx = 0; + } + if (idx == start_idx) { + return ENOENT; + } + } +} + +void *htable_get(const struct htable *htable, const void *key) +{ + uint32_t idx; + + if (htable_get_internal(htable, key, &idx)) { + return NULL; + } + return htable->elem[idx].val; +} + +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val) +{ + uint32_t hole, i; + const void *nkey; + + if (htable_get_internal(htable, key, &hole)) { + *found_key = NULL; + *found_val = NULL; + return; + } + i = hole; + htable->used--; + // We need to maintain the compactness invariant used in + // htable_get_internal. This invariant specifies that the entries for any + // given key are never separated by NULLs (although they may be separated + // by entries for other keys.) + while (1) { + i++; + if (i == htable->capacity) { + i = 0; + } + nkey = htable->elem[i].key; + if (!nkey) { + *found_key = htable->elem[hole].key; + *found_val = htable->elem[hole].val; + htable->elem[hole].key = NULL; + htable->elem[hole].val = NULL; + return; + } else if (htable->eq_fun(key, nkey)) { + htable->elem[hole].key = htable->elem[i].key; + htable->elem[hole].val = htable->elem[i].val; + hole = i; + } + } +} + +uint32_t htable_used(const struct htable *htable) +{ + return htable->used; +} + +uint32_t htable_capacity(const struct htable *htable) +{ + return htable->capacity; +} + +uint32_t ht_hash_string(const void *str, uint32_t max) +{ + const char *s = str; + uint32_t hash = 0; + + while (*s) { + hash = (hash * 31) + *s; + s++; + } + return hash % max; +} + +int ht_compare_string(const void *a, const void *b) +{ + return strcmp(a, b) == 0; +} + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_2_10/common/htable.h b/libhdfs/hdfs_2_10/common/htable.h new file mode 100644 index 0000000..33f1229 --- /dev/null +++ b/libhdfs/hdfs_2_10/common/htable.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HADOOP_CORE_COMMON_HASH_TABLE +#define HADOOP_CORE_COMMON_HASH_TABLE + +#include +#include +#include + +#define HTABLE_MIN_SIZE 4 + +struct htable; + +/** + * An HTable hash function. + * + * @param key The key. + * @param capacity The total capacity. + * + * @return The hash slot. Must be less than the capacity. + */ +typedef uint32_t (*htable_hash_fn_t)(const void *key, uint32_t capacity); + +/** + * An HTable equality function. Compares two keys. + * + * @param a First key. + * @param b Second key. + * + * @return nonzero if the keys are equal. + */ +typedef int (*htable_eq_fn_t)(const void *a, const void *b); + +/** + * Allocate a new hash table. + * + * @param capacity The minimum suggested starting capacity. + * @param hash_fun The hash function to use in this hash table. + * @param eq_fun The equals function to use in this hash table. + * + * @return The new hash table on success; NULL on OOM. + */ +struct htable *htable_alloc(uint32_t capacity, htable_hash_fn_t hash_fun, + htable_eq_fn_t eq_fun); + +typedef void (*visitor_fn_t)(void *ctx, void *key, void *val); + +/** + * Visit all of the entries in the hash table. + * + * @param htable The hash table. + * @param fun The callback function to invoke on each key and value. + * @param ctx Context pointer to pass to the callback. + */ +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx); + +/** + * Free the hash table. + * + * It is up the calling code to ensure that the keys and values inside the + * table are de-allocated, if that is necessary. + * + * @param htable The hash table. + */ +void htable_free(struct htable *htable); + +/** + * Add an entry to the hash table. + * + * @param htable The hash table. + * @param key The key to add. This cannot be NULL. + * @param fun The value to add. This cannot be NULL. + * + * @return 0 on success; + * EEXIST if the value already exists in the table; + * ENOMEM if there is not enough memory to add the element. + * EFBIG if the hash table has too many entries to fit in 32 + * bits. + */ +int htable_put(struct htable *htable, void *key, void *val); + +/** + * Get an entry from the hash table. + * + * @param htable The hash table. + * @param key The key to find. + * + * @return NULL if there is no such entry; the entry otherwise. + */ +void *htable_get(const struct htable *htable, const void *key); + +/** + * Get an entry from the hash table and remove it. + * + * @param htable The hash table. + * @param key The key for the entry find and remove. + * @param found_key (out param) NULL if the entry was not found; the found key + * otherwise. + * @param found_val (out param) NULL if the entry was not found; the found + * value otherwise. + */ +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val); + +/** + * Get the number of entries used in the hash table. + * + * @param htable The hash table. + * + * @return The number of entries used in the hash table. + */ +uint32_t htable_used(const struct htable *htable); + +/** + * Get the capacity of the hash table. + * + * @param htable The hash table. + * + * @return The capacity of the hash table. + */ +uint32_t htable_capacity(const struct htable *htable); + +/** + * Hash a string. + * + * @param str The string. + * @param max Maximum hash value + * + * @return A number less than max. + */ +uint32_t ht_hash_string(const void *str, uint32_t max); + +/** + * Compare two strings. + * + * @param a The first string. + * @param b The second string. + * + * @return 1 if the strings are identical; 0 otherwise. + */ +int ht_compare_string(const void *a, const void *b); + +#endif + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_2_10/exception.c b/libhdfs/hdfs_2_10/exception.c new file mode 100644 index 0000000..35e9d2d --- /dev/null +++ b/libhdfs/hdfs_2_10/exception.c @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "java.lang.UnsupportedOperationException", + 0, + ENOTSUP, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jstring jStr = NULL; + jvalue jVal; + jthrowable jthr; + const char *stackTrace; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "ExceptionUtils::getStackTrace error.)\n", className); + destroyLocalReference(env, jthr); + } else { + jStr = jVal.l; + stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL); + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "GetStringUTFChars error.)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + (*env)->ReleaseStringUTFChars(env, jStr, stackTrace); + } + } + } + destroyLocalReference(env, jStr); + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_2_10/exception.h b/libhdfs/hdfs_2_10/exception.h new file mode 100644 index 0000000..5fa7fa6 --- /dev/null +++ b/libhdfs/hdfs_2_10/exception.h @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * PrintExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Print out information about the pending exception and free it. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/libhdfs/hdfs_2_10/hdfs.c b/libhdfs/hdfs_2_10/hdfs.c new file mode 100644 index 0000000..e3cd580 --- /dev/null +++ b/libhdfs/hdfs_2_10/hdfs.c @@ -0,0 +1,3591 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define HADOOP_FS_DATA_INPUT_STREAM "org/apache/hadoop/fs/FSDataInputStream" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) +#define HDFS_FILE_SUPPORTS_DIRECT_PREAD (1<<0) + +/** + * Reads bytes using the read(ByteBuffer) API. By using Java + * DirectByteBuffers we can avoid copying the bytes from kernel space into + * user space. + */ +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); + +/** + * Reads bytes using the read(long, ByteBuffer) API. By using Java + * DirectByteBuffers we can avoid copying the bytes from kernel space into + * user space. + */ +tSize preadDirect(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, + tSize length); + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +int hdfsFileClearReadStatistics(hdfsFile file) +{ + jthrowable jthr; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return EINTERNAL; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "clearReadStatistics", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileClearReadStatistics: clearReadStatistics failed"); + goto done; + } + ret = 0; +done: + if (ret) { + errno = ret; + return ret; + } + return 0; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsFileUsesDirectPread(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD); +} + +void hdfsFileDisableDirectPread(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_PREAD; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + struct hdfsStreamBuilder *bld = hdfsStreamBuilderAlloc(fs, path, flags); + if (bufferSize != 0) { + hdfsStreamBuilderSetBufferSize(bld, bufferSize); + } + if (replication != 0) { + hdfsStreamBuilderSetReplication(bld, replication); + } + if (blockSize != 0) { + hdfsStreamBuilderSetDefaultBlockSize(bld, blockSize); + } + return hdfsStreamBuilderBuild(bld); +} + +struct hdfsStreamBuilder { + hdfsFS fs; + int flags; + int32_t bufferSize; + int16_t replication; + int64_t defaultBlockSize; + char path[1]; +}; + +struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs, + const char *path, int flags) +{ + int path_len = strlen(path); + struct hdfsStreamBuilder *bld; + + // sizeof(hdfsStreamBuilder->path) includes one byte for the string + // terminator + bld = malloc(sizeof(struct hdfsStreamBuilder) + path_len); + if (!bld) { + errno = ENOMEM; + return NULL; + } + bld->fs = fs; + bld->flags = flags; + bld->bufferSize = 0; + bld->replication = 0; + bld->defaultBlockSize = 0; + memcpy(bld->path, path, path_len); + bld->path[path_len] = '\0'; + return bld; +} + +void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld) +{ + free(bld); +} + +int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld, + int32_t bufferSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->bufferSize = bufferSize; + return 0; +} + +int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld, + int16_t replication) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->replication = replication; + return 0; +} + +int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld, + int64_t defaultBlockSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->defaultBlockSize = defaultBlockSize; + return 0; +} + +/** + * Delegates to FsDataInputStream#hasCapability(String). Used to check if a + * given input stream supports certain methods, such as + * ByteBufferReadable#read(ByteBuffer). + * + * @param jFile the FsDataInputStream to call hasCapability on + * @param capability the name of the capability to query; for a full list of + * possible values see StreamCapabilities + * + * @return true if the given jFile has the given capability, false otherwise + * + * @see org.apache.hadoop.fs.StreamCapabilities + */ +static int hdfsHasStreamCapability(jobject jFile, + const char *capability) { + int ret = 0; + jthrowable jthr = NULL; + jvalue jVal; + jstring jCapabilityString = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return 0; + } + + jthr = newJavaStr(env, capability, &jCapabilityString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHasStreamCapability(%s): newJavaStr", capability); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFile, + HADOOP_FS_DATA_INPUT_STREAM, "hasCapability", "(Ljava/lang/String;)Z", + jCapabilityString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHasStreamCapability(%s): FSDataInputStream#hasCapability", + capability); + goto done; + } + +done: + destroyLocalReference(env, jthr); + destroyLocalReference(env, jCapabilityString); + if (ret) { + errno = ret; + return 0; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, + int32_t bufferSize, int16_t replication, int64_t blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL, jCapabilityString = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Check the StreamCapabilities of jFile to see if we can do direct + // reads + if (hdfsHasStreamCapability(jFile, "in:readbytebuffer")) { + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } + + // Check the StreamCapabilities of jFile to see if we can do direct + // preads + if (hdfsHasStreamCapability(jFile, "in:preadbytebuffer")) { + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_PREAD; + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + destroyLocalReference(env, jCapabilityString); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld) +{ + hdfsFile file = hdfsOpenFileImpl(bld->fs, bld->path, bld->flags, + bld->bufferSize, bld->replication, bld->defaultBlockSize); + int prevErrno = errno; + hdfsStreamBuilderFree(bld); + errno = prevErrno; + return file; +} + +int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) +{ + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + jobject jPath = NULL; + + JNIEnv *env = getJNIEnv(); + + if (!env) { + errno = EINTERNAL; + return -1; + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): constructNewObjectOfPath", path); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "truncate", JMETHOD2(JPARAM(HADOOP_PATH), "J", "Z"), + jPath, newlength); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): FileSystem#truncate", path); + return -1; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +int hdfsUnbufferFile(hdfsFile file) +{ + int ret; + jthrowable jthr; + JNIEnv *env = getJNIEnv(); + + if (!env) { + ret = EINTERNAL; + goto done; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = ENOTSUP; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, HADOOP_ISTRM, + "unbuffer", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + HADOOP_ISTRM "#unbuffer failed:"); + goto done; + } + ret = 0; + +done: + errno = ret; + return ret; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + const char *interface; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + interface = (file->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +/** + * If the underlying stream supports the ByteBufferReadable interface then + * this method will transparently use read(ByteBuffer). This can help + * improve performance as it avoids unnecessary copies between the kernel + * space, the Java process space, and the C process space. + */ +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +/** + * If the underlying stream supports the ByteBufferPositionedReadable + * interface then this method will transparently use read(long, ByteBuffer). + * This can help improve performance as it avoids unnecessary copies between + * the kernel space, the Java process space, and the C process space. + */ +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD) { + return preadDirect(fs, f, position, buffer, length); + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize preadDirect(hdfsFS fs, hdfsFile f, tOffset position, void* buffer, + tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer buf = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(position, buf); + + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, + HADOOP_FS_DATA_INPUT_STREAM, "read", "(JLjava/nio/ByteBuffer;)I", + position, bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "preadDirect: FSDataInputStream#read"); + return -1; + } + // Reached EOF, return 0 + if (jVal.i < 0) { + return 0; + } + // 0 bytes read, return error + if (jVal.i == 0) { + errno = EINTR; + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + const char *interface; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + interface = (f->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + errno = 0; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + + + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_2.10.h b/libhdfs/hdfs_2_10/include/hdfs/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_2.10.h rename to libhdfs/hdfs_2_10/include/hdfs/hdfs.h diff --git a/libhdfs/hdfs_2_10/jni_helper.c b/libhdfs/hdfs_2_10/jni_helper.c new file mode 100644 index 0000000..50d9681 --- /dev/null +++ b/libhdfs/hdfs_2_10/jni_helper.c @@ -0,0 +1,595 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" +#include "platform.h" +#include "common/htable.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include + +static struct htable *gClassRefHTable = NULL; + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + jmethodID mid = 0; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jthrowable jthr = NULL; + jclass local_clazz = NULL; + jclass clazz = NULL; + int ret; + + mutexLock(&hdfsHashMutex); + if (!gClassRefHTable) { + gClassRefHTable = htable_alloc(MAX_HASH_TABLE_ELEM, ht_hash_string, + ht_compare_string); + if (!gClassRefHTable) { + jthr = newRuntimeError(env, "htable_alloc failed\n"); + goto done; + } + } + clazz = htable_get(gClassRefHTable, className); + if (clazz) { + *out = clazz; + goto done; + } + local_clazz = (*env)->FindClass(env,className); + if (!local_clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clazz = (*env)->NewGlobalRef(env, local_clazz); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + ret = htable_put(gClassRefHTable, (void*)className, clazz); + if (ret) { + jthr = newRuntimeError(env, "htable_put failed with error " + "code %d\n", ret); + goto done; + } + *out = clazz; + jthr = NULL; +done: + mutexUnlock(&hdfsHashMutex); + (*env)->DeleteLocalRef(env, local_clazz); + if (jthr && clazz) { + (*env)->DeleteGlobalRef(env, clazz); + } + return jthr; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + JNIEnv *env; + THREAD_LOCAL_STORAGE_GET_QUICK(); + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&env)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (env) { + mutexUnlock(&jvmMutex); + return env; + } + + env = getGlobalJNIEnv(); + mutexUnlock(&jvmMutex); + if (!env) { + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + return NULL; + } + if (threadLocalStorageSet(env)) { + return NULL; + } + THREAD_LOCAL_STORAGE_SET_QUICK(env); + return env; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_2_10/jni_helper.h b/libhdfs/hdfs_2_10/jni_helper.h new file mode 100644 index 0000000..90accc7 --- /dev/null +++ b/libhdfs/hdfs_2_10/jni_helper.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_2_10/os/mutexes.h b/libhdfs/hdfs_2_10/os/mutexes.h new file mode 100644 index 0000000..da30bf4 --- /dev/null +++ b/libhdfs/hdfs_2_10/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting the class reference hash table. */ +extern mutex hdfsHashMutex; + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/libhdfs/hdfs_2_10/os/posix/mutexes.c b/libhdfs/hdfs_2_10/os/posix/mutexes.c new file mode 100644 index 0000000..c4c2f26 --- /dev/null +++ b/libhdfs/hdfs_2_10/os/posix/mutexes.c @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +mutex jvmMutex = PTHREAD_MUTEX_INITIALIZER; + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/libhdfs/hdfs_2_10/os/posix/platform.h b/libhdfs/hdfs_2_10/os/posix/platform.h new file mode 100644 index 0000000..c63bbf9 --- /dev/null +++ b/libhdfs/hdfs_2_10/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/libhdfs/hdfs_2_10/os/posix/thread.c b/libhdfs/hdfs_2_10/os/posix/thread.c new file mode 100644 index 0000000..af0c61f --- /dev/null +++ b/libhdfs/hdfs_2_10/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/libhdfs/hdfs_2_10/os/posix/thread_local_storage.c b/libhdfs/hdfs_2_10/os/posix/thread_local_storage.c new file mode 100644 index 0000000..2f70e2c --- /dev/null +++ b/libhdfs/hdfs_2_10/os/posix/thread_local_storage.c @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +static void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + JNIEnv *env = v; + jint ret; + + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } +} + +int threadLocalStorageGet(JNIEnv **env) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *env = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(JNIEnv *env) +{ + int ret = pthread_setspecific(gTlsKey, env); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(env); + } + return ret; +} diff --git a/libhdfs/hdfs_2_10/os/thread.h b/libhdfs/hdfs_2_10/os/thread.h new file mode 100644 index 0000000..ae425d3 --- /dev/null +++ b/libhdfs/hdfs_2_10/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/libhdfs/hdfs_2_10/os/thread_local_storage.h b/libhdfs/hdfs_2_10/os/thread_local_storage.h new file mode 100644 index 0000000..a40d567 --- /dev/null +++ b/libhdfs/hdfs_2_10/os/thread_local_storage.h @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK() \ + static __thread JNIEnv *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + return quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(env) \ + { \ + quickTlsEnv = (env); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK() + #define THREAD_LOCAL_STORAGE_SET_QUICK(env) +#endif + +/** + * Gets the JNIEnv in thread-local storage for the current thread. If the call + * succeeds, and there is a JNIEnv associated with this thread, then returns 0 + * and populates env. If the call succeeds, but there is no JNIEnv associated + * with this thread, then returns 0 and sets JNIEnv to NULL. If the call fails, + * then returns non-zero. Only one thread at a time may execute this function. + * The caller is responsible for enforcing mutual exclusion. + * + * @param env JNIEnv out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(JNIEnv **env); + +/** + * Sets the JNIEnv in thread-local storage for the current thread. + * + * @param env JNIEnv to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(JNIEnv *env); + +#endif diff --git a/libhdfs/hdfs_2_10/os/windows/inttypes.h b/libhdfs/hdfs_2_10/os/windows/inttypes.h new file mode 100644 index 0000000..a520d15 --- /dev/null +++ b/libhdfs/hdfs_2_10/os/windows/inttypes.h @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_INTTYPES_H +#define LIBHDFS_INTTYPES_H + +/* On Windows, inttypes.h does not exist, so manually define what we need. */ + +#define PRId64 "I64d" +#define PRIu64 "I64u" +typedef unsigned __int64 uint64_t; + +#endif diff --git a/libhdfs/hdfs_2_10/os/windows/mutexes.c b/libhdfs/hdfs_2_10/os/windows/mutexes.c new file mode 100644 index 0000000..875f033 --- /dev/null +++ b/libhdfs/hdfs_2_10/os/windows/mutexes.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include + +mutex hdfsHashMutex; +mutex jvmMutex; + +/** + * Unfortunately, there is no simple static initializer for a critical section. + * Instead, the API requires calling InitializeCriticalSection. Since libhdfs + * lacks an explicit initialization function, there is no obvious existing place + * for the InitializeCriticalSection calls. To work around this, we define an + * initialization function and instruct the linker to set a pointer to that + * function as a user-defined global initializer. See discussion of CRT + * Initialization: + * http://msdn.microsoft.com/en-us/library/bb918180.aspx + */ +static void __cdecl initializeMutexes(void) { + InitializeCriticalSection(&hdfsHashMutex); + InitializeCriticalSection(&jvmMutex); +} +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) +const void (__cdecl *pInitialize)(void) = initializeMutexes; + +int mutexLock(mutex *m) { + EnterCriticalSection(m); + return 0; +} + +int mutexUnlock(mutex *m) { + LeaveCriticalSection(m); + return 0; +} diff --git a/libhdfs/hdfs_2_10/os/windows/platform.h b/libhdfs/hdfs_2_10/os/windows/platform.h new file mode 100644 index 0000000..9eedfde --- /dev/null +++ b/libhdfs/hdfs_2_10/os/windows/platform.h @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include +#include +#include + +/* + * O_ACCMODE defined to match Linux definition. + */ +#ifndef O_ACCMODE +#define O_ACCMODE 0x0003 +#endif + +/* + * Windows has a different name for its maximum path length constant. + */ +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +/* + * Windows does not define EDQUOT and ESTALE in errno.h. The closest equivalents + * are these constants from winsock.h. + */ +#ifndef EDQUOT +#define EDQUOT WSAEDQUOT +#endif + +#ifndef ESTALE +#define ESTALE WSAESTALE +#endif + +/* + * gcc-style type-checked format arguments are not supported on Windows, so just + * stub this macro. + */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) + +/* + * Define macros for various string formatting functions not defined on Windows. + * Where possible, we reroute to one of the secure CRT variants. On Windows, + * the preprocessor does support variadic macros, even though they weren't + * defined until C99. + */ +#define snprintf(str, size, format, ...) \ + _snprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) +#define strncpy(dest, src, n) \ + strncpy_s((dest), (n), (src), _TRUNCATE) +#define strtok_r(str, delim, saveptr) \ + strtok_s((str), (delim), (saveptr)) +#define vsnprintf(str, size, format, ...) \ + vsnprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) + +/* + * Mutex data type defined as Windows CRITICAL_SECTION. A critical section (not + * Windows mutex) is used, because libhdfs only needs synchronization of multiple + * threads within a single process, not synchronization across process + * boundaries. + */ +typedef CRITICAL_SECTION mutex; + +/* + * Thread data type defined as HANDLE to a Windows thread. + */ +typedef HANDLE threadId; + +#endif diff --git a/libhdfs/hdfs_2_10/os/windows/thread.c b/libhdfs/hdfs_2_10/os/windows/thread.c new file mode 100644 index 0000000..f5cc2a7 --- /dev/null +++ b/libhdfs/hdfs_2_10/os/windows/thread.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by CreateThread. + * + * @param toRun thread to run + * @return DWORD result of running thread (always 0) + */ +static DWORD WINAPI runThread(LPVOID toRun) { + const thread *t = toRun; + t->start(t->arg); + return 0; +} + +int threadCreate(thread *t) { + DWORD ret = 0; + HANDLE h; + h = CreateThread(NULL, 0, runThread, t, 0, NULL); + if (h) { + t->id = h; + } else { + ret = GetLastError(); + fprintf(stderr, "threadCreate: CreateThread failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + DWORD ret = WaitForSingleObject(t->id, INFINITE); + switch (ret) { + case WAIT_OBJECT_0: + break; + case WAIT_FAILED: + ret = GetLastError(); + fprintf(stderr, "threadJoin: WaitForSingleObject failed with error %d\n", + ret); + break; + default: + fprintf(stderr, "threadJoin: WaitForSingleObject unexpected error %d\n", + ret); + break; + } + return ret; +} diff --git a/libhdfs/hdfs_2_10/os/windows/thread_local_storage.c b/libhdfs/hdfs_2_10/os/windows/thread_local_storage.c new file mode 100644 index 0000000..4c415e1 --- /dev/null +++ b/libhdfs/hdfs_2_10/os/windows/thread_local_storage.c @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static DWORD gTlsIndex = TLS_OUT_OF_INDEXES; + +/** + * If the current thread has a JNIEnv in thread-local storage, then detaches the + * current thread from the JVM. + */ +static void detachCurrentThreadFromJvm() +{ + JNIEnv *env = NULL; + JavaVM *vm; + jint ret; + if (threadLocalStorageGet(&env) || !env) { + return; + } + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, + "detachCurrentThreadFromJvm: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } +} + +/** + * Unlike pthreads, the Windows API does not seem to provide a convenient way to + * hook a callback onto thread shutdown. However, the Windows portable + * executable format does define a concept of thread-local storage callbacks. + * Here, we define a function and instruct the linker to set a pointer to that + * function in the segment for thread-local storage callbacks. See page 85 of + * Microsoft Portable Executable and Common Object File Format Specification: + * http://msdn.microsoft.com/en-us/gg463119.aspx + * This technique only works for implicit linking (OS loads DLL on demand), not + * for explicit linking (user code calls LoadLibrary directly). This effectively + * means that we have a known limitation: libhdfs may not work correctly if a + * Windows application attempts to use it via explicit linking. + * + * @param h module handle + * @param reason the reason for calling the callback + * @param pv reserved, unused + */ +static void NTAPI tlsCallback(PVOID h, DWORD reason, PVOID pv) +{ + DWORD tlsIndex; + switch (reason) { + case DLL_THREAD_DETACH: + detachCurrentThreadFromJvm(); + break; + case DLL_PROCESS_DETACH: + detachCurrentThreadFromJvm(); + tlsIndex = gTlsIndex; + gTlsIndex = TLS_OUT_OF_INDEXES; + if (!TlsFree(tlsIndex)) { + fprintf(stderr, "tlsCallback: TlsFree failed with error %d\n", + GetLastError()); + } + break; + default: + break; + } +} + +/* + * A variable named _tls_used contains the TLS directory, which contains a list + * of pointers to callback functions. Normally, the linker won't retain this + * variable unless the executable has implicit thread-local variables, defined + * using the __declspec(thread) extended storage-class modifier. libhdfs + * doesn't use __declspec(thread), and we have no guarantee that the executable + * linked to libhdfs will use __declspec(thread). By forcing the linker to + * reference _tls_used, we guarantee that the binary retains the TLS directory. + * See Microsoft Visual Studio 10.0/VC/crt/src/tlssup.c . + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#endif + +/* + * We must retain a pointer to the callback function. Force the linker to keep + * this symbol, even though it appears that nothing in our source code uses it. + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:pTlsCallback") +#else +#pragma comment(linker, "/INCLUDE:_pTlsCallback") +#endif + +/* + * Define constant pointer to our callback, and tell the linker to pin it into + * the TLS directory so that it receives thread callbacks. Use external linkage + * to protect against the linker discarding the seemingly unused symbol. + */ +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK pTlsCallback; +const PIMAGE_TLS_CALLBACK pTlsCallback = tlsCallback; +#pragma const_seg() + +int threadLocalStorageGet(JNIEnv **env) +{ + LPVOID tls; + DWORD ret; + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + gTlsIndex = TlsAlloc(); + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + fprintf(stderr, + "threadLocalStorageGet: TlsAlloc failed with error %d\n", + TLS_OUT_OF_INDEXES); + return TLS_OUT_OF_INDEXES; + } + } + tls = TlsGetValue(gTlsIndex); + if (tls) { + *env = tls; + return 0; + } else { + ret = GetLastError(); + if (ERROR_SUCCESS == ret) { + /* Thread-local storage contains NULL, because we haven't set it yet. */ + *env = NULL; + return 0; + } else { + /* + * The API call failed. According to documentation, TlsGetValue cannot + * fail as long as the index is a valid index from a successful TlsAlloc + * call. This error handling is purely defensive. + */ + fprintf(stderr, + "threadLocalStorageGet: TlsGetValue failed with error %d\n", ret); + return ret; + } + } +} + +int threadLocalStorageSet(JNIEnv *env) +{ + DWORD ret = 0; + if (!TlsSetValue(gTlsIndex, (LPVOID)env)) { + ret = GetLastError(); + fprintf(stderr, + "threadLocalStorageSet: TlsSetValue failed with error %d\n", + ret); + detachCurrentThreadFromJvm(env); + } + return ret; +} diff --git a/libhdfs/hdfs_2_10/os/windows/unistd.h b/libhdfs/hdfs_2_10/os/windows/unistd.h new file mode 100644 index 0000000..b82ce48 --- /dev/null +++ b/libhdfs/hdfs_2_10/os/windows/unistd.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_UNISTD_H +#define LIBHDFS_UNISTD_H + +/* On Windows, unistd.h does not exist, so manually define what we need. */ + +#include /* Declares getpid(). */ +#include + +/* Re-route sleep to Sleep, converting units from seconds to milliseconds. */ +#define sleep(seconds) Sleep((seconds) * 1000) +#endif diff --git a/libhdfs/hdfs_2_2/exception.c b/libhdfs/hdfs_2_2/exception.c new file mode 100644 index 0000000..6a50c98 --- /dev/null +++ b/libhdfs/hdfs_2_2/exception.c @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" + +#include +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + .name = "java/io/FileNotFoundException", + .noPrintFlag = NOPRINT_EXC_FILE_NOT_FOUND, + .excErrno = ENOENT, + }, + { + .name = "org/apache/hadoop/security/AccessControlException", + .noPrintFlag = NOPRINT_EXC_ACCESS_CONTROL, + .excErrno = EACCES, + }, + { + .name = "org/apache/hadoop/fs/UnresolvedLinkException", + .noPrintFlag = NOPRINT_EXC_UNRESOLVED_LINK, + .excErrno = ENOLINK, + }, + { + .name = "org/apache/hadoop/fs/ParentNotDirectoryException", + .noPrintFlag = NOPRINT_EXC_PARENT_NOT_DIRECTORY, + .excErrno = ENOTDIR, + }, + { + .name = "java/lang/IllegalArgumentException", + .noPrintFlag = NOPRINT_EXC_ILLEGAL_ARGUMENT, + .excErrno = EINVAL, + }, + { + .name = "java/lang/OutOfMemoryError", + .noPrintFlag = 0, + .excErrno = ENOMEM, + }, + +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jstring jStr = NULL; + jvalue jVal; + jthrowable jthr; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "ExceptionUtils::getStackTrace error.)\n", className); + destroyLocalReference(env, jthr); + } else { + jStr = jVal.l; + const char *stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL); + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "GetStringUTFChars error.)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + (*env)->ReleaseStringUTFChars(env, jStr, stackTrace); + } + } + } + destroyLocalReference(env, jStr); + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_2_2/exception.h b/libhdfs/hdfs_2_2/exception.h new file mode 100644 index 0000000..e361513 --- /dev/null +++ b/libhdfs/hdfs_2_2/exception.h @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * PrintExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + */ + +#include +#include + +#include +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) __attribute__((format(printf, 4, 5))); + +/** + * Print out information about the pending exception and free it. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) __attribute__((format(printf, 3, 4))); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); + +#endif diff --git a/libhdfs/hdfs_2_2/hdfs.c b/libhdfs/hdfs_2_2/hdfs.c new file mode 100644 index 0000000..2782434 --- /dev/null +++ b/libhdfs/hdfs_2_2/hdfs.c @@ -0,0 +1,2758 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" + +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + UNINITIALIZED = 0, + INPUT = 1, + OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == INPUT); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +/** + * Set a configuration value. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +static jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + HADOOP_CONF, "set", JMETHOD2(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING), JAVA_VOID), + jkey, jvalue); + if (jthr) + goto done; +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jvalue); + return jthr; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char* host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char* host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char* host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char* host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = rindex(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + int accmode = flags & O_ACCMODE; + + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + /* The hadoop java api/signature */ + const char* method = NULL; + const char* signature = NULL; + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jint jBufferSize = bufferSize; + jshort jReplication = replication; + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? INPUT : OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Caught exception + jthrowable jthr; + + //Sanity check + if (!file || file->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //The interface whose 'close' method to be called + const char* interface = (file->type == INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + const char *interfaceShortName = (file->type == INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jInputStream; + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jInputStream; + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + jvalue jVal; + jthrowable jthr; + + //Read the requisite bytes + jobject bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jbyteArray jbWarray; + jthrowable jthr; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != INPUT) { + errno = EBADF; + return -1; + } + + jobject jInputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jobject jStream = f->file; + const char* interface = (f->type == INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jvalue jVal; + jthrowable jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jobject jInputStream = f->file; + jvalue jVal; + jthrowable jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char* src, hdfsFS dstFS, + const char* dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char* path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jboolean jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jvalue jVal; + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jvalue jVal; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char* path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + + // construct jPerm = FsPermission.createImmutable(short mode); + jshort jmode = mode; + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + const tTime NO_CHANGE = -1; + jlong jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jlong jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +char*** +hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jobject jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jsize jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + const char *hostName; + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getDefaultBlockSize() + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getStatus + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + jobject fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getStatus + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + jobject fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + + + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + const char *cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + const char* cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mOwner = strdup(cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + + const char* cGroupName; + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + jsize i; + jobject tmpStat; + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthrowable jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + hdfsFileInfo *fileInfo; + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + + + + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_2.2.h b/libhdfs/hdfs_2_2/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_2.2.h rename to libhdfs/hdfs_2_2/hdfs.h diff --git a/libhdfs/hdfs_2_2/jni_helper.c b/libhdfs/hdfs_2_2/jni_helper.c new file mode 100644 index 0000000..c768c9c --- /dev/null +++ b/libhdfs/hdfs_2_2/jni_helper.c @@ -0,0 +1,610 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" + +#include +#include + +static pthread_mutex_t hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t jvmMutex = PTHREAD_MUTEX_INITIALIZER; +static volatile int hashTableInited = 0; + +#define LOCK_HASH_TABLE() pthread_mutex_lock(&hdfsHashMutex) +#define UNLOCK_HASH_TABLE() pthread_mutex_unlock(&hdfsHashMutex) + + +/** The Native return types that methods could return */ +#define VOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** Pthreads thread-local storage for each library thread. */ +struct hdfsTls { + JNIEnv *env; +}; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +static void hdfsThreadDestructor(void *v) +{ + struct hdfsTls *tls = v; + JavaVM *vm; + JNIEnv *env = tls->env; + jint ret; + + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with " + "error %d\n", ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } + free(tls); +} + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +static int hashTableInit(void) +{ + if (!hashTableInited) { + LOCK_HASH_TABLE(); + if (!hashTableInited) { + if (hcreate(MAX_HASH_TABLE_ELEM) == 0) { + fprintf(stderr, "error creating hashtable, <%d>: %s\n", + errno, strerror(errno)); + UNLOCK_HASH_TABLE(); + return 0; + } + hashTableInited = 1; + } + UNLOCK_HASH_TABLE(); + } + return 1; +} + + +static int insertEntryIntoTable(const char *key, void *data) +{ + ENTRY e, *ep; + if (key == NULL || data == NULL) { + return 0; + } + if (! hashTableInit()) { + return -1; + } + e.data = data; + e.key = (char*)key; + LOCK_HASH_TABLE(); + ep = hsearch(e, ENTER); + UNLOCK_HASH_TABLE(); + if (ep == NULL) { + fprintf(stderr, "warn adding key (%s) to hash table, <%d>: %s\n", + key, errno, strerror(errno)); + } + return 0; +} + + + +static void* searchEntryFromTable(const char *key) +{ + ENTRY e,*ep; + if (key == NULL) { + return NULL; + } + hashTableInit(); + e.key = (char*)key; + LOCK_HASH_TABLE(); + ep = hsearch(e, FIND); + UNLOCK_HASH_TABLE(); + if (ep != NULL) { + return ep->data; + } + return NULL; +} + + + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == VOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jmethodID mid = 0; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jclass clsLocalRef; + jclass cls = searchEntryFromTable(className); + if (cls) { + *out = cls; + return NULL; + } + clsLocalRef = (*env)->FindClass(env,className); + if (clsLocalRef == NULL) { + return getPendingExceptionAndClear(env); + } + cls = (*env)->NewGlobalRef(env, clsLocalRef); + if (cls == NULL) { + (*env)->DeleteLocalRef(env, clsLocalRef); + return getPendingExceptionAndClear(env); + } + (*env)->DeleteLocalRef(env, clsLocalRef); + insertEntryIntoTable(className, cls); + *out = cls; + return NULL; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + const jsize vmBufLength = 1; + JavaVM* vmBuf[vmBufLength]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), vmBufLength, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + char *hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + char *optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + int noArgs = 1; + char *hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + JavaVMOption options[noArgs]; + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + JavaVMInitArgs vm_args; + JavaVM *vm; + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + JavaVM* vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + JNIEnv *env; + struct hdfsTls *tls; + int ret; + +#ifdef HAVE_BETTER_TLS + static __thread struct hdfsTls *quickTls = NULL; + if (quickTls) + return quickTls->env; +#endif + pthread_mutex_lock(&jvmMutex); + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + pthread_mutex_unlock(&jvmMutex); + fprintf(stderr, "getJNIEnv: pthread_key_create failed with " + "error %d\n", ret); + return NULL; + } + gTlsKeyInitialized = 1; + } + tls = pthread_getspecific(gTlsKey); + if (tls) { + pthread_mutex_unlock(&jvmMutex); + return tls->env; + } + + env = getGlobalJNIEnv(); + pthread_mutex_unlock(&jvmMutex); + if (!env) { + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + return NULL; + } + tls = calloc(1, sizeof(struct hdfsTls)); + if (!tls) { + fprintf(stderr, "getJNIEnv: OOM allocating %zd bytes\n", + sizeof(struct hdfsTls)); + return NULL; + } + tls->env = env; + ret = pthread_setspecific(gTlsKey, tls); + if (ret) { + fprintf(stderr, "getJNIEnv: pthread_setspecific failed with " + "error code %d\n", ret); + hdfsThreadDestructor(tls); + return NULL; + } +#ifdef HAVE_BETTER_TLS + quickTls = tls; +#endif + return env; +} + diff --git a/libhdfs/hdfs_2_2/jni_helper.h b/libhdfs/hdfs_2_2/jni_helper.h new file mode 100644 index 0000000..f37dea7 --- /dev/null +++ b/libhdfs/hdfs_2_2/jni_helper.h @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_2_3/exception.c b/libhdfs/hdfs_2_3/exception.c new file mode 100644 index 0000000..d7e1720 --- /dev/null +++ b/libhdfs/hdfs_2_3/exception.c @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" + +#include +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + .name = "java.io.FileNotFoundException", + .noPrintFlag = NOPRINT_EXC_FILE_NOT_FOUND, + .excErrno = ENOENT, + }, + { + .name = "org.apache.hadoop.security.AccessControlException", + .noPrintFlag = NOPRINT_EXC_ACCESS_CONTROL, + .excErrno = EACCES, + }, + { + .name = "org.apache.hadoop.fs.UnresolvedLinkException", + .noPrintFlag = NOPRINT_EXC_UNRESOLVED_LINK, + .excErrno = ENOLINK, + }, + { + .name = "org.apache.hadoop.fs.ParentNotDirectoryException", + .noPrintFlag = NOPRINT_EXC_PARENT_NOT_DIRECTORY, + .excErrno = ENOTDIR, + }, + { + .name = "java.lang.IllegalArgumentException", + .noPrintFlag = NOPRINT_EXC_ILLEGAL_ARGUMENT, + .excErrno = EINVAL, + }, + { + .name = "java.lang.OutOfMemoryError", + .noPrintFlag = 0, + .excErrno = ENOMEM, + }, + { + .name = "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + .noPrintFlag = 0, + .excErrno = EROFS, + }, + { + .name = "org.apache.hadoop.fs.FileAlreadyExistsException", + .noPrintFlag = 0, + .excErrno = EEXIST, + }, + { + .name = "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + .noPrintFlag = 0, + .excErrno = EDQUOT, + }, + { + .name = "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + .noPrintFlag = 0, + .excErrno = ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jstring jStr = NULL; + jvalue jVal; + jthrowable jthr; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "ExceptionUtils::getStackTrace error.)\n", className); + destroyLocalReference(env, jthr); + } else { + jStr = jVal.l; + const char *stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL); + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "GetStringUTFChars error.)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + (*env)->ReleaseStringUTFChars(env, jStr, stackTrace); + } + } + } + destroyLocalReference(env, jStr); + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_2_3/exception.h b/libhdfs/hdfs_2_3/exception.h new file mode 100644 index 0000000..e361513 --- /dev/null +++ b/libhdfs/hdfs_2_3/exception.h @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * PrintExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + */ + +#include +#include + +#include +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) __attribute__((format(printf, 4, 5))); + +/** + * Print out information about the pending exception and free it. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) __attribute__((format(printf, 3, 4))); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); + +#endif diff --git a/libhdfs/hdfs_2_3/hdfs.c b/libhdfs/hdfs_2_3/hdfs.c new file mode 100644 index 0000000..e519f35 --- /dev/null +++ b/libhdfs/hdfs_2_3/hdfs.c @@ -0,0 +1,3144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" + +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + UNINITIALIZED = 0, + INPUT = 1, + OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == INPUT); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char* host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char* host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char* host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char* host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = rindex(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + int accmode = flags & O_ACCMODE; + + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + /* The hadoop java api/signature */ + const char* method = NULL; + const char* signature = NULL; + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jint jBufferSize = bufferSize; + jshort jReplication = replication; + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? INPUT : OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Caught exception + jthrowable jthr; + + //Sanity check + if (!file || file->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //The interface whose 'close' method to be called + const char* interface = (file->type == INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + const char *interfaceShortName = (file->type == INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jInputStream; + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jInputStream; + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + jvalue jVal; + jthrowable jthr; + + //Read the requisite bytes + jobject bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jbyteArray jbWarray; + jthrowable jthr; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != INPUT) { + errno = EBADF; + return -1; + } + + jobject jInputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jobject jStream = f->file; + const char* interface = (f->type == INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jvalue jVal; + jthrowable jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jobject jInputStream = f->file; + jvalue jVal; + jthrowable jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char* src, hdfsFS dstFS, + const char* dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char* path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jboolean jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jvalue jVal; + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jvalue jVal; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char* path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + + // construct jPerm = FsPermission.createImmutable(short mode); + jshort jmode = mode; + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + const tTime NO_CHANGE = -1; + jlong jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jlong jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jobject jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jsize jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + const char *hostName; + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getDefaultBlockSize() + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getStatus + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + jobject fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getStatus + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + jobject fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + + + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + const char *cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + const char* cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mOwner = strdup(cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + + const char* cGroupName; + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + jsize i; + jobject tmpStat; + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthrowable jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + hdfsFileInfo *fileInfo; + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + + + + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_2.3.h b/libhdfs/hdfs_2_3/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_2.3.h rename to libhdfs/hdfs_2_3/hdfs.h diff --git a/libhdfs/hdfs_2_3/jni_helper.c b/libhdfs/hdfs_2_3/jni_helper.c new file mode 100644 index 0000000..878289f --- /dev/null +++ b/libhdfs/hdfs_2_3/jni_helper.c @@ -0,0 +1,680 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" + +#include +#include + +static pthread_mutex_t hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t jvmMutex = PTHREAD_MUTEX_INITIALIZER; +static volatile int hashTableInited = 0; + +#define LOCK_HASH_TABLE() pthread_mutex_lock(&hdfsHashMutex) +#define UNLOCK_HASH_TABLE() pthread_mutex_unlock(&hdfsHashMutex) + + +/** The Native return types that methods could return */ +#define VOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** Pthreads thread-local storage for each library thread. */ +struct hdfsTls { + JNIEnv *env; +}; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +static void hdfsThreadDestructor(void *v) +{ + struct hdfsTls *tls = v; + JavaVM *vm; + JNIEnv *env = tls->env; + jint ret; + + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with " + "error %d\n", ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } + free(tls); +} + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +static int hashTableInit(void) +{ + if (!hashTableInited) { + LOCK_HASH_TABLE(); + if (!hashTableInited) { + if (hcreate(MAX_HASH_TABLE_ELEM) == 0) { + fprintf(stderr, "error creating hashtable, <%d>: %s\n", + errno, strerror(errno)); + UNLOCK_HASH_TABLE(); + return 0; + } + hashTableInited = 1; + } + UNLOCK_HASH_TABLE(); + } + return 1; +} + + +static int insertEntryIntoTable(const char *key, void *data) +{ + ENTRY e, *ep; + if (key == NULL || data == NULL) { + return 0; + } + if (! hashTableInit()) { + return -1; + } + e.data = data; + e.key = (char*)key; + LOCK_HASH_TABLE(); + ep = hsearch(e, ENTER); + UNLOCK_HASH_TABLE(); + if (ep == NULL) { + fprintf(stderr, "warn adding key (%s) to hash table, <%d>: %s\n", + key, errno, strerror(errno)); + } + return 0; +} + + + +static void* searchEntryFromTable(const char *key) +{ + ENTRY e,*ep; + if (key == NULL) { + return NULL; + } + hashTableInit(); + e.key = (char*)key; + LOCK_HASH_TABLE(); + ep = hsearch(e, FIND); + UNLOCK_HASH_TABLE(); + if (ep != NULL) { + return ep->data; + } + return NULL; +} + + + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == VOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jmethodID mid = 0; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jclass clsLocalRef; + jclass cls = searchEntryFromTable(className); + if (cls) { + *out = cls; + return NULL; + } + clsLocalRef = (*env)->FindClass(env,className); + if (clsLocalRef == NULL) { + return getPendingExceptionAndClear(env); + } + cls = (*env)->NewGlobalRef(env, clsLocalRef); + if (cls == NULL) { + (*env)->DeleteLocalRef(env, clsLocalRef); + return getPendingExceptionAndClear(env); + } + (*env)->DeleteLocalRef(env, clsLocalRef); + insertEntryIntoTable(className, cls); + *out = cls; + return NULL; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + const jsize vmBufLength = 1; + JavaVM* vmBuf[vmBufLength]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), vmBufLength, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + char *hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + char *optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + int noArgs = 1; + char *hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + JavaVMOption options[noArgs]; + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + JavaVMInitArgs vm_args; + JavaVM *vm; + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + JavaVM* vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + JNIEnv *env; + struct hdfsTls *tls; + int ret; + +#ifdef HAVE_BETTER_TLS + static __thread struct hdfsTls *quickTls = NULL; + if (quickTls) + return quickTls->env; +#endif + pthread_mutex_lock(&jvmMutex); + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + pthread_mutex_unlock(&jvmMutex); + fprintf(stderr, "getJNIEnv: pthread_key_create failed with " + "error %d\n", ret); + return NULL; + } + gTlsKeyInitialized = 1; + } + tls = pthread_getspecific(gTlsKey); + if (tls) { + pthread_mutex_unlock(&jvmMutex); + return tls->env; + } + + env = getGlobalJNIEnv(); + pthread_mutex_unlock(&jvmMutex); + if (!env) { + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + return NULL; + } + tls = calloc(1, sizeof(struct hdfsTls)); + if (!tls) { + fprintf(stderr, "getJNIEnv: OOM allocating %zd bytes\n", + sizeof(struct hdfsTls)); + return NULL; + } + tls->env = env; + ret = pthread_setspecific(gTlsKey, tls); + if (ret) { + fprintf(stderr, "getJNIEnv: pthread_setspecific failed with " + "error code %d\n", ret); + hdfsThreadDestructor(tls); + return NULL; + } +#ifdef HAVE_BETTER_TLS + quickTls = tls; +#endif + return env; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_2_3/jni_helper.h b/libhdfs/hdfs_2_3/jni_helper.h new file mode 100644 index 0000000..c09f6a3 --- /dev/null +++ b/libhdfs/hdfs_2_3/jni_helper.h @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_2_4/exception.c b/libhdfs/hdfs_2_4/exception.c new file mode 100644 index 0000000..d7e1720 --- /dev/null +++ b/libhdfs/hdfs_2_4/exception.c @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" + +#include +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + .name = "java.io.FileNotFoundException", + .noPrintFlag = NOPRINT_EXC_FILE_NOT_FOUND, + .excErrno = ENOENT, + }, + { + .name = "org.apache.hadoop.security.AccessControlException", + .noPrintFlag = NOPRINT_EXC_ACCESS_CONTROL, + .excErrno = EACCES, + }, + { + .name = "org.apache.hadoop.fs.UnresolvedLinkException", + .noPrintFlag = NOPRINT_EXC_UNRESOLVED_LINK, + .excErrno = ENOLINK, + }, + { + .name = "org.apache.hadoop.fs.ParentNotDirectoryException", + .noPrintFlag = NOPRINT_EXC_PARENT_NOT_DIRECTORY, + .excErrno = ENOTDIR, + }, + { + .name = "java.lang.IllegalArgumentException", + .noPrintFlag = NOPRINT_EXC_ILLEGAL_ARGUMENT, + .excErrno = EINVAL, + }, + { + .name = "java.lang.OutOfMemoryError", + .noPrintFlag = 0, + .excErrno = ENOMEM, + }, + { + .name = "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + .noPrintFlag = 0, + .excErrno = EROFS, + }, + { + .name = "org.apache.hadoop.fs.FileAlreadyExistsException", + .noPrintFlag = 0, + .excErrno = EEXIST, + }, + { + .name = "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + .noPrintFlag = 0, + .excErrno = EDQUOT, + }, + { + .name = "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + .noPrintFlag = 0, + .excErrno = ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jstring jStr = NULL; + jvalue jVal; + jthrowable jthr; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "ExceptionUtils::getStackTrace error.)\n", className); + destroyLocalReference(env, jthr); + } else { + jStr = jVal.l; + const char *stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL); + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "GetStringUTFChars error.)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + (*env)->ReleaseStringUTFChars(env, jStr, stackTrace); + } + } + } + destroyLocalReference(env, jStr); + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_2_4/exception.h b/libhdfs/hdfs_2_4/exception.h new file mode 100644 index 0000000..e361513 --- /dev/null +++ b/libhdfs/hdfs_2_4/exception.h @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * PrintExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + */ + +#include +#include + +#include +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) __attribute__((format(printf, 4, 5))); + +/** + * Print out information about the pending exception and free it. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) __attribute__((format(printf, 3, 4))); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); + +#endif diff --git a/libhdfs/hdfs_2_4/hdfs.c b/libhdfs/hdfs_2_4/hdfs.c new file mode 100644 index 0000000..e519f35 --- /dev/null +++ b/libhdfs/hdfs_2_4/hdfs.c @@ -0,0 +1,3144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" + +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + UNINITIALIZED = 0, + INPUT = 1, + OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == INPUT); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char* host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char* host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char* host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char* host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = rindex(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + int accmode = flags & O_ACCMODE; + + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + /* The hadoop java api/signature */ + const char* method = NULL; + const char* signature = NULL; + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jint jBufferSize = bufferSize; + jshort jReplication = replication; + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? INPUT : OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Caught exception + jthrowable jthr; + + //Sanity check + if (!file || file->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //The interface whose 'close' method to be called + const char* interface = (file->type == INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + const char *interfaceShortName = (file->type == INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jInputStream; + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jInputStream; + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + jvalue jVal; + jthrowable jthr; + + //Read the requisite bytes + jobject bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jbyteArray jbWarray; + jthrowable jthr; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != INPUT) { + errno = EBADF; + return -1; + } + + jobject jInputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jobject jStream = f->file; + const char* interface = (f->type == INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jvalue jVal; + jthrowable jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jobject jInputStream = f->file; + jvalue jVal; + jthrowable jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char* src, hdfsFS dstFS, + const char* dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char* path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jboolean jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jvalue jVal; + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jvalue jVal; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char* path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + + // construct jPerm = FsPermission.createImmutable(short mode); + jshort jmode = mode; + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + const tTime NO_CHANGE = -1; + jlong jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jlong jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jobject jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jsize jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + const char *hostName; + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getDefaultBlockSize() + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getStatus + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + jobject fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getStatus + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + jobject fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + + + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + const char *cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + const char* cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mOwner = strdup(cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + + const char* cGroupName; + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + jsize i; + jobject tmpStat; + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthrowable jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + hdfsFileInfo *fileInfo; + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + + + + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_2.4.h b/libhdfs/hdfs_2_4/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_2.4.h rename to libhdfs/hdfs_2_4/hdfs.h diff --git a/libhdfs/hdfs_2_4/jni_helper.c b/libhdfs/hdfs_2_4/jni_helper.c new file mode 100644 index 0000000..878289f --- /dev/null +++ b/libhdfs/hdfs_2_4/jni_helper.c @@ -0,0 +1,680 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" + +#include +#include + +static pthread_mutex_t hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t jvmMutex = PTHREAD_MUTEX_INITIALIZER; +static volatile int hashTableInited = 0; + +#define LOCK_HASH_TABLE() pthread_mutex_lock(&hdfsHashMutex) +#define UNLOCK_HASH_TABLE() pthread_mutex_unlock(&hdfsHashMutex) + + +/** The Native return types that methods could return */ +#define VOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** Pthreads thread-local storage for each library thread. */ +struct hdfsTls { + JNIEnv *env; +}; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +static void hdfsThreadDestructor(void *v) +{ + struct hdfsTls *tls = v; + JavaVM *vm; + JNIEnv *env = tls->env; + jint ret; + + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with " + "error %d\n", ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } + free(tls); +} + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +static int hashTableInit(void) +{ + if (!hashTableInited) { + LOCK_HASH_TABLE(); + if (!hashTableInited) { + if (hcreate(MAX_HASH_TABLE_ELEM) == 0) { + fprintf(stderr, "error creating hashtable, <%d>: %s\n", + errno, strerror(errno)); + UNLOCK_HASH_TABLE(); + return 0; + } + hashTableInited = 1; + } + UNLOCK_HASH_TABLE(); + } + return 1; +} + + +static int insertEntryIntoTable(const char *key, void *data) +{ + ENTRY e, *ep; + if (key == NULL || data == NULL) { + return 0; + } + if (! hashTableInit()) { + return -1; + } + e.data = data; + e.key = (char*)key; + LOCK_HASH_TABLE(); + ep = hsearch(e, ENTER); + UNLOCK_HASH_TABLE(); + if (ep == NULL) { + fprintf(stderr, "warn adding key (%s) to hash table, <%d>: %s\n", + key, errno, strerror(errno)); + } + return 0; +} + + + +static void* searchEntryFromTable(const char *key) +{ + ENTRY e,*ep; + if (key == NULL) { + return NULL; + } + hashTableInit(); + e.key = (char*)key; + LOCK_HASH_TABLE(); + ep = hsearch(e, FIND); + UNLOCK_HASH_TABLE(); + if (ep != NULL) { + return ep->data; + } + return NULL; +} + + + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == VOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jmethodID mid = 0; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jclass clsLocalRef; + jclass cls = searchEntryFromTable(className); + if (cls) { + *out = cls; + return NULL; + } + clsLocalRef = (*env)->FindClass(env,className); + if (clsLocalRef == NULL) { + return getPendingExceptionAndClear(env); + } + cls = (*env)->NewGlobalRef(env, clsLocalRef); + if (cls == NULL) { + (*env)->DeleteLocalRef(env, clsLocalRef); + return getPendingExceptionAndClear(env); + } + (*env)->DeleteLocalRef(env, clsLocalRef); + insertEntryIntoTable(className, cls); + *out = cls; + return NULL; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + const jsize vmBufLength = 1; + JavaVM* vmBuf[vmBufLength]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), vmBufLength, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + char *hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + char *optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + int noArgs = 1; + char *hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + JavaVMOption options[noArgs]; + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + JavaVMInitArgs vm_args; + JavaVM *vm; + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + JavaVM* vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + JNIEnv *env; + struct hdfsTls *tls; + int ret; + +#ifdef HAVE_BETTER_TLS + static __thread struct hdfsTls *quickTls = NULL; + if (quickTls) + return quickTls->env; +#endif + pthread_mutex_lock(&jvmMutex); + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + pthread_mutex_unlock(&jvmMutex); + fprintf(stderr, "getJNIEnv: pthread_key_create failed with " + "error %d\n", ret); + return NULL; + } + gTlsKeyInitialized = 1; + } + tls = pthread_getspecific(gTlsKey); + if (tls) { + pthread_mutex_unlock(&jvmMutex); + return tls->env; + } + + env = getGlobalJNIEnv(); + pthread_mutex_unlock(&jvmMutex); + if (!env) { + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + return NULL; + } + tls = calloc(1, sizeof(struct hdfsTls)); + if (!tls) { + fprintf(stderr, "getJNIEnv: OOM allocating %zd bytes\n", + sizeof(struct hdfsTls)); + return NULL; + } + tls->env = env; + ret = pthread_setspecific(gTlsKey, tls); + if (ret) { + fprintf(stderr, "getJNIEnv: pthread_setspecific failed with " + "error code %d\n", ret); + hdfsThreadDestructor(tls); + return NULL; + } +#ifdef HAVE_BETTER_TLS + quickTls = tls; +#endif + return env; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_2_4/jni_helper.h b/libhdfs/hdfs_2_4/jni_helper.h new file mode 100644 index 0000000..c09f6a3 --- /dev/null +++ b/libhdfs/hdfs_2_4/jni_helper.h @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_2_5/exception.c b/libhdfs/hdfs_2_5/exception.c new file mode 100644 index 0000000..d7e1720 --- /dev/null +++ b/libhdfs/hdfs_2_5/exception.c @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" + +#include +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + .name = "java.io.FileNotFoundException", + .noPrintFlag = NOPRINT_EXC_FILE_NOT_FOUND, + .excErrno = ENOENT, + }, + { + .name = "org.apache.hadoop.security.AccessControlException", + .noPrintFlag = NOPRINT_EXC_ACCESS_CONTROL, + .excErrno = EACCES, + }, + { + .name = "org.apache.hadoop.fs.UnresolvedLinkException", + .noPrintFlag = NOPRINT_EXC_UNRESOLVED_LINK, + .excErrno = ENOLINK, + }, + { + .name = "org.apache.hadoop.fs.ParentNotDirectoryException", + .noPrintFlag = NOPRINT_EXC_PARENT_NOT_DIRECTORY, + .excErrno = ENOTDIR, + }, + { + .name = "java.lang.IllegalArgumentException", + .noPrintFlag = NOPRINT_EXC_ILLEGAL_ARGUMENT, + .excErrno = EINVAL, + }, + { + .name = "java.lang.OutOfMemoryError", + .noPrintFlag = 0, + .excErrno = ENOMEM, + }, + { + .name = "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + .noPrintFlag = 0, + .excErrno = EROFS, + }, + { + .name = "org.apache.hadoop.fs.FileAlreadyExistsException", + .noPrintFlag = 0, + .excErrno = EEXIST, + }, + { + .name = "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + .noPrintFlag = 0, + .excErrno = EDQUOT, + }, + { + .name = "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + .noPrintFlag = 0, + .excErrno = ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jstring jStr = NULL; + jvalue jVal; + jthrowable jthr; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "ExceptionUtils::getStackTrace error.)\n", className); + destroyLocalReference(env, jthr); + } else { + jStr = jVal.l; + const char *stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL); + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "GetStringUTFChars error.)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + (*env)->ReleaseStringUTFChars(env, jStr, stackTrace); + } + } + } + destroyLocalReference(env, jStr); + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_2_5/exception.h b/libhdfs/hdfs_2_5/exception.h new file mode 100644 index 0000000..e361513 --- /dev/null +++ b/libhdfs/hdfs_2_5/exception.h @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * PrintExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + */ + +#include +#include + +#include +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) __attribute__((format(printf, 4, 5))); + +/** + * Print out information about the pending exception and free it. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) __attribute__((format(printf, 3, 4))); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); + +#endif diff --git a/libhdfs/hdfs_2_5/hdfs.c b/libhdfs/hdfs_2_5/hdfs.c new file mode 100644 index 0000000..e519f35 --- /dev/null +++ b/libhdfs/hdfs_2_5/hdfs.c @@ -0,0 +1,3144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" + +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + UNINITIALIZED = 0, + INPUT = 1, + OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == INPUT); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char* host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char* host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char* host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char* host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = rindex(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + int accmode = flags & O_ACCMODE; + + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + /* The hadoop java api/signature */ + const char* method = NULL; + const char* signature = NULL; + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jint jBufferSize = bufferSize; + jshort jReplication = replication; + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? INPUT : OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Caught exception + jthrowable jthr; + + //Sanity check + if (!file || file->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //The interface whose 'close' method to be called + const char* interface = (file->type == INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + const char *interfaceShortName = (file->type == INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jInputStream; + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jInputStream; + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + jvalue jVal; + jthrowable jthr; + + //Read the requisite bytes + jobject bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jbyteArray jbWarray; + jthrowable jthr; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != INPUT) { + errno = EBADF; + return -1; + } + + jobject jInputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jobject jStream = f->file; + const char* interface = (f->type == INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jvalue jVal; + jthrowable jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != OUTPUT) { + errno = EBADF; + return -1; + } + + jobject jOutputStream = f->file; + jthrowable jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jobject jInputStream = f->file; + jvalue jVal; + jthrowable jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char* src, hdfsFS dstFS, + const char* dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char* path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jboolean jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jvalue jVal; + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + jthrowable jthr; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jvalue jVal; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char* path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + + // construct jPerm = FsPermission.createImmutable(short mode); + jshort jmode = mode; + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + const tTime NO_CHANGE = -1; + jlong jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jlong jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char* path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jobject jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jsize jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + const char *hostName; + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getDefaultBlockSize() + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getStatus + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + jobject fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jobject jFS = (jobject)fs; + + //FileSystem#getStatus + jvalue jVal; + jthrowable jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + jobject fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + + + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + const char *cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + const char* cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mOwner = strdup(cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + + const char* cGroupName; + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char* path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + jsize i; + jobject tmpStat; + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + jobject jFS = (jobject)fs; + + //Create an object of org.apache.hadoop.fs.Path + jobject jPath; + jthrowable jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + hdfsFileInfo *fileInfo; + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + + + + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_2.5.h b/libhdfs/hdfs_2_5/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_2.5.h rename to libhdfs/hdfs_2_5/hdfs.h diff --git a/libhdfs/hdfs_2_5/jni_helper.c b/libhdfs/hdfs_2_5/jni_helper.c new file mode 100644 index 0000000..878289f --- /dev/null +++ b/libhdfs/hdfs_2_5/jni_helper.c @@ -0,0 +1,680 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" + +#include +#include + +static pthread_mutex_t hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t jvmMutex = PTHREAD_MUTEX_INITIALIZER; +static volatile int hashTableInited = 0; + +#define LOCK_HASH_TABLE() pthread_mutex_lock(&hdfsHashMutex) +#define UNLOCK_HASH_TABLE() pthread_mutex_unlock(&hdfsHashMutex) + + +/** The Native return types that methods could return */ +#define VOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** Pthreads thread-local storage for each library thread. */ +struct hdfsTls { + JNIEnv *env; +}; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +static void hdfsThreadDestructor(void *v) +{ + struct hdfsTls *tls = v; + JavaVM *vm; + JNIEnv *env = tls->env; + jint ret; + + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with " + "error %d\n", ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } + free(tls); +} + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +static int hashTableInit(void) +{ + if (!hashTableInited) { + LOCK_HASH_TABLE(); + if (!hashTableInited) { + if (hcreate(MAX_HASH_TABLE_ELEM) == 0) { + fprintf(stderr, "error creating hashtable, <%d>: %s\n", + errno, strerror(errno)); + UNLOCK_HASH_TABLE(); + return 0; + } + hashTableInited = 1; + } + UNLOCK_HASH_TABLE(); + } + return 1; +} + + +static int insertEntryIntoTable(const char *key, void *data) +{ + ENTRY e, *ep; + if (key == NULL || data == NULL) { + return 0; + } + if (! hashTableInit()) { + return -1; + } + e.data = data; + e.key = (char*)key; + LOCK_HASH_TABLE(); + ep = hsearch(e, ENTER); + UNLOCK_HASH_TABLE(); + if (ep == NULL) { + fprintf(stderr, "warn adding key (%s) to hash table, <%d>: %s\n", + key, errno, strerror(errno)); + } + return 0; +} + + + +static void* searchEntryFromTable(const char *key) +{ + ENTRY e,*ep; + if (key == NULL) { + return NULL; + } + hashTableInit(); + e.key = (char*)key; + LOCK_HASH_TABLE(); + ep = hsearch(e, FIND); + UNLOCK_HASH_TABLE(); + if (ep != NULL) { + return ep->data; + } + return NULL; +} + + + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == VOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jmethodID mid = 0; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jclass clsLocalRef; + jclass cls = searchEntryFromTable(className); + if (cls) { + *out = cls; + return NULL; + } + clsLocalRef = (*env)->FindClass(env,className); + if (clsLocalRef == NULL) { + return getPendingExceptionAndClear(env); + } + cls = (*env)->NewGlobalRef(env, clsLocalRef); + if (cls == NULL) { + (*env)->DeleteLocalRef(env, clsLocalRef); + return getPendingExceptionAndClear(env); + } + (*env)->DeleteLocalRef(env, clsLocalRef); + insertEntryIntoTable(className, cls); + *out = cls; + return NULL; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + const jsize vmBufLength = 1; + JavaVM* vmBuf[vmBufLength]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), vmBufLength, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + char *hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + char *optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + int noArgs = 1; + char *hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + JavaVMOption options[noArgs]; + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + JavaVMInitArgs vm_args; + JavaVM *vm; + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + JavaVM* vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + JNIEnv *env; + struct hdfsTls *tls; + int ret; + +#ifdef HAVE_BETTER_TLS + static __thread struct hdfsTls *quickTls = NULL; + if (quickTls) + return quickTls->env; +#endif + pthread_mutex_lock(&jvmMutex); + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + pthread_mutex_unlock(&jvmMutex); + fprintf(stderr, "getJNIEnv: pthread_key_create failed with " + "error %d\n", ret); + return NULL; + } + gTlsKeyInitialized = 1; + } + tls = pthread_getspecific(gTlsKey); + if (tls) { + pthread_mutex_unlock(&jvmMutex); + return tls->env; + } + + env = getGlobalJNIEnv(); + pthread_mutex_unlock(&jvmMutex); + if (!env) { + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + return NULL; + } + tls = calloc(1, sizeof(struct hdfsTls)); + if (!tls) { + fprintf(stderr, "getJNIEnv: OOM allocating %zd bytes\n", + sizeof(struct hdfsTls)); + return NULL; + } + tls->env = env; + ret = pthread_setspecific(gTlsKey, tls); + if (ret) { + fprintf(stderr, "getJNIEnv: pthread_setspecific failed with " + "error code %d\n", ret); + hdfsThreadDestructor(tls); + return NULL; + } +#ifdef HAVE_BETTER_TLS + quickTls = tls; +#endif + return env; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_2_5/jni_helper.h b/libhdfs/hdfs_2_5/jni_helper.h new file mode 100644 index 0000000..c09f6a3 --- /dev/null +++ b/libhdfs/hdfs_2_5/jni_helper.h @@ -0,0 +1,163 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_2_6/common/htable.c b/libhdfs/hdfs_2_6/common/htable.c new file mode 100644 index 0000000..8d3d3c5 --- /dev/null +++ b/libhdfs/hdfs_2_6/common/htable.c @@ -0,0 +1,271 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/htable.h" + +#include +#include +#include +#include +#include + +struct htable_pair { + void *key; + void *val; +}; + +/** + * A hash table which uses linear probing. + */ +struct htable { + uint32_t capacity; + uint32_t used; + htable_hash_fn_t hash_fun; + htable_eq_fn_t eq_fun; + struct htable_pair *elem; +}; + +/** + * An internal function for inserting a value into the hash table. + * + * Note: this function assumes that you have made enough space in the table. + * + * @param nelem The new element to insert. + * @param capacity The capacity of the hash table. + * @param hash_fun The hash function to use. + * @param key The key to insert. + * @param val The value to insert. + */ +static void htable_insert_internal(struct htable_pair *nelem, + uint32_t capacity, htable_hash_fn_t hash_fun, void *key, + void *val) +{ + uint32_t i; + + i = hash_fun(key, capacity); + while (1) { + if (!nelem[i].key) { + nelem[i].key = key; + nelem[i].val = val; + return; + } + i++; + if (i == capacity) { + i = 0; + } + } +} + +static int htable_realloc(struct htable *htable, uint32_t new_capacity) +{ + struct htable_pair *nelem; + uint32_t i, old_capacity = htable->capacity; + htable_hash_fn_t hash_fun = htable->hash_fun; + + nelem = calloc(new_capacity, sizeof(struct htable_pair)); + if (!nelem) { + return ENOMEM; + } + for (i = 0; i < old_capacity; i++) { + struct htable_pair *pair = htable->elem + i; + htable_insert_internal(nelem, new_capacity, hash_fun, + pair->key, pair->val); + } + free(htable->elem); + htable->elem = nelem; + htable->capacity = new_capacity; + return 0; +} + +struct htable *htable_alloc(uint32_t size, + htable_hash_fn_t hash_fun, htable_eq_fn_t eq_fun) +{ + struct htable *htable; + + htable = calloc(1, sizeof(*htable)); + if (!htable) { + return NULL; + } + size = (size + 1) >> 1; + size = size << 1; + if (size < HTABLE_MIN_SIZE) { + size = HTABLE_MIN_SIZE; + } + htable->hash_fun = hash_fun; + htable->eq_fun = eq_fun; + htable->used = 0; + if (htable_realloc(htable, size)) { + free(htable); + return NULL; + } + return htable; +} + +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx) +{ + uint32_t i; + + for (i = 0; i != htable->capacity; ++i) { + struct htable_pair *elem = htable->elem + i; + if (elem->key) { + fun(ctx, elem->key, elem->val); + } + } +} + +void htable_free(struct htable *htable) +{ + if (htable) { + free(htable->elem); + free(htable); + } +} + +int htable_put(struct htable *htable, void *key, void *val) +{ + int ret; + uint32_t nused; + + // NULL is not a valid key value. + // This helps us implement htable_get_internal efficiently, since we know + // that we can stop when we encounter the first NULL key. + if (!key) { + return EINVAL; + } + // NULL is not a valid value. Otherwise the results of htable_get would + // be confusing (does a NULL return mean entry not found, or that the + // entry was found and was NULL?) + if (!val) { + return EINVAL; + } + // Re-hash if we have used more than half of the hash table + nused = htable->used + 1; + if (nused >= (htable->capacity / 2)) { + ret = htable_realloc(htable, htable->capacity * 2); + if (ret) + return ret; + } + htable_insert_internal(htable->elem, htable->capacity, + htable->hash_fun, key, val); + htable->used++; + return 0; +} + +static int htable_get_internal(const struct htable *htable, + const void *key, uint32_t *out) +{ + uint32_t start_idx, idx; + + start_idx = htable->hash_fun(key, htable->capacity); + idx = start_idx; + while (1) { + struct htable_pair *pair = htable->elem + idx; + if (!pair->key) { + // We always maintain the invariant that the entries corresponding + // to a given key are stored in a contiguous block, not separated + // by any NULLs. So if we encounter a NULL, our search is over. + return ENOENT; + } else if (htable->eq_fun(pair->key, key)) { + *out = idx; + return 0; + } + idx++; + if (idx == htable->capacity) { + idx = 0; + } + if (idx == start_idx) { + return ENOENT; + } + } +} + +void *htable_get(const struct htable *htable, const void *key) +{ + uint32_t idx; + + if (htable_get_internal(htable, key, &idx)) { + return NULL; + } + return htable->elem[idx].val; +} + +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val) +{ + uint32_t hole, i; + const void *nkey; + + if (htable_get_internal(htable, key, &hole)) { + *found_key = NULL; + *found_val = NULL; + return; + } + i = hole; + htable->used--; + // We need to maintain the compactness invariant used in + // htable_get_internal. This invariant specifies that the entries for any + // given key are never separated by NULLs (although they may be separated + // by entries for other keys.) + while (1) { + i++; + if (i == htable->capacity) { + i = 0; + } + nkey = htable->elem[i].key; + if (!nkey) { + *found_key = htable->elem[hole].key; + *found_val = htable->elem[hole].val; + htable->elem[hole].key = NULL; + htable->elem[hole].val = NULL; + return; + } else if (htable->eq_fun(key, nkey)) { + htable->elem[hole].key = htable->elem[i].key; + htable->elem[hole].val = htable->elem[i].val; + hole = i; + } + } +} + +uint32_t htable_used(const struct htable *htable) +{ + return htable->used; +} + +uint32_t htable_capacity(const struct htable *htable) +{ + return htable->capacity; +} + +uint32_t ht_hash_string(const void *str, uint32_t max) +{ + const char *s = str; + uint32_t hash = 0; + + while (*s) { + hash = (hash * 31) + *s; + s++; + } + return hash % max; +} + +int ht_compare_string(const void *a, const void *b) +{ + return strcmp(a, b) == 0; +} + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_2_6/common/htable.h b/libhdfs/hdfs_2_6/common/htable.h new file mode 100644 index 0000000..33f1229 --- /dev/null +++ b/libhdfs/hdfs_2_6/common/htable.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HADOOP_CORE_COMMON_HASH_TABLE +#define HADOOP_CORE_COMMON_HASH_TABLE + +#include +#include +#include + +#define HTABLE_MIN_SIZE 4 + +struct htable; + +/** + * An HTable hash function. + * + * @param key The key. + * @param capacity The total capacity. + * + * @return The hash slot. Must be less than the capacity. + */ +typedef uint32_t (*htable_hash_fn_t)(const void *key, uint32_t capacity); + +/** + * An HTable equality function. Compares two keys. + * + * @param a First key. + * @param b Second key. + * + * @return nonzero if the keys are equal. + */ +typedef int (*htable_eq_fn_t)(const void *a, const void *b); + +/** + * Allocate a new hash table. + * + * @param capacity The minimum suggested starting capacity. + * @param hash_fun The hash function to use in this hash table. + * @param eq_fun The equals function to use in this hash table. + * + * @return The new hash table on success; NULL on OOM. + */ +struct htable *htable_alloc(uint32_t capacity, htable_hash_fn_t hash_fun, + htable_eq_fn_t eq_fun); + +typedef void (*visitor_fn_t)(void *ctx, void *key, void *val); + +/** + * Visit all of the entries in the hash table. + * + * @param htable The hash table. + * @param fun The callback function to invoke on each key and value. + * @param ctx Context pointer to pass to the callback. + */ +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx); + +/** + * Free the hash table. + * + * It is up the calling code to ensure that the keys and values inside the + * table are de-allocated, if that is necessary. + * + * @param htable The hash table. + */ +void htable_free(struct htable *htable); + +/** + * Add an entry to the hash table. + * + * @param htable The hash table. + * @param key The key to add. This cannot be NULL. + * @param fun The value to add. This cannot be NULL. + * + * @return 0 on success; + * EEXIST if the value already exists in the table; + * ENOMEM if there is not enough memory to add the element. + * EFBIG if the hash table has too many entries to fit in 32 + * bits. + */ +int htable_put(struct htable *htable, void *key, void *val); + +/** + * Get an entry from the hash table. + * + * @param htable The hash table. + * @param key The key to find. + * + * @return NULL if there is no such entry; the entry otherwise. + */ +void *htable_get(const struct htable *htable, const void *key); + +/** + * Get an entry from the hash table and remove it. + * + * @param htable The hash table. + * @param key The key for the entry find and remove. + * @param found_key (out param) NULL if the entry was not found; the found key + * otherwise. + * @param found_val (out param) NULL if the entry was not found; the found + * value otherwise. + */ +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val); + +/** + * Get the number of entries used in the hash table. + * + * @param htable The hash table. + * + * @return The number of entries used in the hash table. + */ +uint32_t htable_used(const struct htable *htable); + +/** + * Get the capacity of the hash table. + * + * @param htable The hash table. + * + * @return The capacity of the hash table. + */ +uint32_t htable_capacity(const struct htable *htable); + +/** + * Hash a string. + * + * @param str The string. + * @param max Maximum hash value + * + * @return A number less than max. + */ +uint32_t ht_hash_string(const void *str, uint32_t max); + +/** + * Compare two strings. + * + * @param a The first string. + * @param b The second string. + * + * @return 1 if the strings are identical; 0 otherwise. + */ +int ht_compare_string(const void *a, const void *b); + +#endif + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_2_6/exception.c b/libhdfs/hdfs_2_6/exception.c new file mode 100644 index 0000000..2373aa7 --- /dev/null +++ b/libhdfs/hdfs_2_6/exception.c @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jstring jStr = NULL; + jvalue jVal; + jthrowable jthr; + const char *stackTrace; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "ExceptionUtils::getStackTrace error.)\n", className); + destroyLocalReference(env, jthr); + } else { + jStr = jVal.l; + stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL); + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "GetStringUTFChars error.)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + (*env)->ReleaseStringUTFChars(env, jStr, stackTrace); + } + } + } + destroyLocalReference(env, jStr); + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_2_6/exception.h b/libhdfs/hdfs_2_6/exception.h new file mode 100644 index 0000000..5fa7fa6 --- /dev/null +++ b/libhdfs/hdfs_2_6/exception.h @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * PrintExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Print out information about the pending exception and free it. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/libhdfs/hdfs_2_6/hdfs.c b/libhdfs/hdfs_2_6/hdfs.c new file mode 100644 index 0000000..21f9c2b --- /dev/null +++ b/libhdfs/hdfs_2_6/hdfs.c @@ -0,0 +1,3245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + const char *interface; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + interface = (file->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + const char *interface; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + interface = (f->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + + + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_2.6.h b/libhdfs/hdfs_2_6/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_2.6.h rename to libhdfs/hdfs_2_6/hdfs.h diff --git a/libhdfs/hdfs_2_6/jni_helper.c b/libhdfs/hdfs_2_6/jni_helper.c new file mode 100644 index 0000000..50d9681 --- /dev/null +++ b/libhdfs/hdfs_2_6/jni_helper.c @@ -0,0 +1,595 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" +#include "platform.h" +#include "common/htable.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include + +static struct htable *gClassRefHTable = NULL; + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + jmethodID mid = 0; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jthrowable jthr = NULL; + jclass local_clazz = NULL; + jclass clazz = NULL; + int ret; + + mutexLock(&hdfsHashMutex); + if (!gClassRefHTable) { + gClassRefHTable = htable_alloc(MAX_HASH_TABLE_ELEM, ht_hash_string, + ht_compare_string); + if (!gClassRefHTable) { + jthr = newRuntimeError(env, "htable_alloc failed\n"); + goto done; + } + } + clazz = htable_get(gClassRefHTable, className); + if (clazz) { + *out = clazz; + goto done; + } + local_clazz = (*env)->FindClass(env,className); + if (!local_clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clazz = (*env)->NewGlobalRef(env, local_clazz); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + ret = htable_put(gClassRefHTable, (void*)className, clazz); + if (ret) { + jthr = newRuntimeError(env, "htable_put failed with error " + "code %d\n", ret); + goto done; + } + *out = clazz; + jthr = NULL; +done: + mutexUnlock(&hdfsHashMutex); + (*env)->DeleteLocalRef(env, local_clazz); + if (jthr && clazz) { + (*env)->DeleteGlobalRef(env, clazz); + } + return jthr; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + JNIEnv *env; + THREAD_LOCAL_STORAGE_GET_QUICK(); + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&env)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (env) { + mutexUnlock(&jvmMutex); + return env; + } + + env = getGlobalJNIEnv(); + mutexUnlock(&jvmMutex); + if (!env) { + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + return NULL; + } + if (threadLocalStorageSet(env)) { + return NULL; + } + THREAD_LOCAL_STORAGE_SET_QUICK(env); + return env; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_2_6/jni_helper.h b/libhdfs/hdfs_2_6/jni_helper.h new file mode 100644 index 0000000..90accc7 --- /dev/null +++ b/libhdfs/hdfs_2_6/jni_helper.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_2_6/os/mutexes.h b/libhdfs/hdfs_2_6/os/mutexes.h new file mode 100644 index 0000000..da30bf4 --- /dev/null +++ b/libhdfs/hdfs_2_6/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting the class reference hash table. */ +extern mutex hdfsHashMutex; + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/libhdfs/hdfs_2_6/os/posix/mutexes.c b/libhdfs/hdfs_2_6/os/posix/mutexes.c new file mode 100644 index 0000000..c4c2f26 --- /dev/null +++ b/libhdfs/hdfs_2_6/os/posix/mutexes.c @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +mutex jvmMutex = PTHREAD_MUTEX_INITIALIZER; + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/libhdfs/hdfs_2_6/os/posix/platform.h b/libhdfs/hdfs_2_6/os/posix/platform.h new file mode 100644 index 0000000..c63bbf9 --- /dev/null +++ b/libhdfs/hdfs_2_6/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/libhdfs/hdfs_2_6/os/posix/thread.c b/libhdfs/hdfs_2_6/os/posix/thread.c new file mode 100644 index 0000000..af0c61f --- /dev/null +++ b/libhdfs/hdfs_2_6/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/libhdfs/hdfs_2_6/os/posix/thread_local_storage.c b/libhdfs/hdfs_2_6/os/posix/thread_local_storage.c new file mode 100644 index 0000000..2f70e2c --- /dev/null +++ b/libhdfs/hdfs_2_6/os/posix/thread_local_storage.c @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +static void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + JNIEnv *env = v; + jint ret; + + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } +} + +int threadLocalStorageGet(JNIEnv **env) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *env = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(JNIEnv *env) +{ + int ret = pthread_setspecific(gTlsKey, env); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(env); + } + return ret; +} diff --git a/libhdfs/hdfs_2_6/os/thread.h b/libhdfs/hdfs_2_6/os/thread.h new file mode 100644 index 0000000..ae425d3 --- /dev/null +++ b/libhdfs/hdfs_2_6/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/libhdfs/hdfs_2_6/os/thread_local_storage.h b/libhdfs/hdfs_2_6/os/thread_local_storage.h new file mode 100644 index 0000000..a40d567 --- /dev/null +++ b/libhdfs/hdfs_2_6/os/thread_local_storage.h @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK() \ + static __thread JNIEnv *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + return quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(env) \ + { \ + quickTlsEnv = (env); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK() + #define THREAD_LOCAL_STORAGE_SET_QUICK(env) +#endif + +/** + * Gets the JNIEnv in thread-local storage for the current thread. If the call + * succeeds, and there is a JNIEnv associated with this thread, then returns 0 + * and populates env. If the call succeeds, but there is no JNIEnv associated + * with this thread, then returns 0 and sets JNIEnv to NULL. If the call fails, + * then returns non-zero. Only one thread at a time may execute this function. + * The caller is responsible for enforcing mutual exclusion. + * + * @param env JNIEnv out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(JNIEnv **env); + +/** + * Sets the JNIEnv in thread-local storage for the current thread. + * + * @param env JNIEnv to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(JNIEnv *env); + +#endif diff --git a/libhdfs/hdfs_2_6/os/windows/inttypes.h b/libhdfs/hdfs_2_6/os/windows/inttypes.h new file mode 100644 index 0000000..a520d15 --- /dev/null +++ b/libhdfs/hdfs_2_6/os/windows/inttypes.h @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_INTTYPES_H +#define LIBHDFS_INTTYPES_H + +/* On Windows, inttypes.h does not exist, so manually define what we need. */ + +#define PRId64 "I64d" +#define PRIu64 "I64u" +typedef unsigned __int64 uint64_t; + +#endif diff --git a/libhdfs/hdfs_2_6/os/windows/mutexes.c b/libhdfs/hdfs_2_6/os/windows/mutexes.c new file mode 100644 index 0000000..875f033 --- /dev/null +++ b/libhdfs/hdfs_2_6/os/windows/mutexes.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include + +mutex hdfsHashMutex; +mutex jvmMutex; + +/** + * Unfortunately, there is no simple static initializer for a critical section. + * Instead, the API requires calling InitializeCriticalSection. Since libhdfs + * lacks an explicit initialization function, there is no obvious existing place + * for the InitializeCriticalSection calls. To work around this, we define an + * initialization function and instruct the linker to set a pointer to that + * function as a user-defined global initializer. See discussion of CRT + * Initialization: + * http://msdn.microsoft.com/en-us/library/bb918180.aspx + */ +static void __cdecl initializeMutexes(void) { + InitializeCriticalSection(&hdfsHashMutex); + InitializeCriticalSection(&jvmMutex); +} +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) +const void (__cdecl *pInitialize)(void) = initializeMutexes; + +int mutexLock(mutex *m) { + EnterCriticalSection(m); + return 0; +} + +int mutexUnlock(mutex *m) { + LeaveCriticalSection(m); + return 0; +} diff --git a/libhdfs/hdfs_2_6/os/windows/platform.h b/libhdfs/hdfs_2_6/os/windows/platform.h new file mode 100644 index 0000000..9eedfde --- /dev/null +++ b/libhdfs/hdfs_2_6/os/windows/platform.h @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include +#include +#include + +/* + * O_ACCMODE defined to match Linux definition. + */ +#ifndef O_ACCMODE +#define O_ACCMODE 0x0003 +#endif + +/* + * Windows has a different name for its maximum path length constant. + */ +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +/* + * Windows does not define EDQUOT and ESTALE in errno.h. The closest equivalents + * are these constants from winsock.h. + */ +#ifndef EDQUOT +#define EDQUOT WSAEDQUOT +#endif + +#ifndef ESTALE +#define ESTALE WSAESTALE +#endif + +/* + * gcc-style type-checked format arguments are not supported on Windows, so just + * stub this macro. + */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) + +/* + * Define macros for various string formatting functions not defined on Windows. + * Where possible, we reroute to one of the secure CRT variants. On Windows, + * the preprocessor does support variadic macros, even though they weren't + * defined until C99. + */ +#define snprintf(str, size, format, ...) \ + _snprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) +#define strncpy(dest, src, n) \ + strncpy_s((dest), (n), (src), _TRUNCATE) +#define strtok_r(str, delim, saveptr) \ + strtok_s((str), (delim), (saveptr)) +#define vsnprintf(str, size, format, ...) \ + vsnprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) + +/* + * Mutex data type defined as Windows CRITICAL_SECTION. A critical section (not + * Windows mutex) is used, because libhdfs only needs synchronization of multiple + * threads within a single process, not synchronization across process + * boundaries. + */ +typedef CRITICAL_SECTION mutex; + +/* + * Thread data type defined as HANDLE to a Windows thread. + */ +typedef HANDLE threadId; + +#endif diff --git a/libhdfs/hdfs_2_6/os/windows/thread.c b/libhdfs/hdfs_2_6/os/windows/thread.c new file mode 100644 index 0000000..90450d8 --- /dev/null +++ b/libhdfs/hdfs_2_6/os/windows/thread.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by CreateThread. + * + * @param toRun thread to run + * @return DWORD result of running thread (always 0) + */ +static DWORD runThread(LPVOID toRun) { + const thread *t = toRun; + t->start(t->arg); + return 0; +} + +int threadCreate(thread *t) { + DWORD ret = 0; + HANDLE h; + h = CreateThread(NULL, 0, runThread, t, 0, NULL); + if (h) { + t->id = h; + } else { + ret = GetLastError(); + fprintf(stderr, "threadCreate: CreateThread failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + DWORD ret = WaitForSingleObject(t->id, INFINITE); + switch (ret) { + case WAIT_OBJECT_0: + break; + case WAIT_FAILED: + ret = GetLastError(); + fprintf(stderr, "threadJoin: WaitForSingleObject failed with error %d\n", + ret); + break; + default: + fprintf(stderr, "threadJoin: WaitForSingleObject unexpected error %d\n", + ret); + break; + } + return ret; +} diff --git a/libhdfs/hdfs_2_6/os/windows/thread_local_storage.c b/libhdfs/hdfs_2_6/os/windows/thread_local_storage.c new file mode 100644 index 0000000..70ad152 --- /dev/null +++ b/libhdfs/hdfs_2_6/os/windows/thread_local_storage.c @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static DWORD gTlsIndex = TLS_OUT_OF_INDEXES; + +/** + * If the current thread has a JNIEnv in thread-local storage, then detaches the + * current thread from the JVM. + */ +static void detachCurrentThreadFromJvm() +{ + JNIEnv *env = NULL; + JavaVM *vm; + jint ret; + if (threadLocalStorageGet(&env) || !env) { + return; + } + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, + "detachCurrentThreadFromJvm: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } +} + +/** + * Unlike pthreads, the Windows API does not seem to provide a convenient way to + * hook a callback onto thread shutdown. However, the Windows portable + * executable format does define a concept of thread-local storage callbacks. + * Here, we define a function and instruct the linker to set a pointer to that + * function in the segment for thread-local storage callbacks. See page 85 of + * Microsoft Portable Executable and Common Object File Format Specification: + * http://msdn.microsoft.com/en-us/gg463119.aspx + * This technique only works for implicit linking (OS loads DLL on demand), not + * for explicit linking (user code calls LoadLibrary directly). This effectively + * means that we have a known limitation: libhdfs may not work correctly if a + * Windows application attempts to use it via explicit linking. + * + * @param h module handle + * @param reason the reason for calling the callback + * @param pv reserved, unused + */ +static void NTAPI tlsCallback(PVOID h, DWORD reason, PVOID pv) +{ + DWORD tlsIndex; + switch (reason) { + case DLL_THREAD_DETACH: + detachCurrentThreadFromJvm(); + break; + case DLL_PROCESS_DETACH: + detachCurrentThreadFromJvm(); + tlsIndex = gTlsIndex; + gTlsIndex = TLS_OUT_OF_INDEXES; + if (!TlsFree(tlsIndex)) { + fprintf(stderr, "tlsCallback: TlsFree failed with error %d\n", + GetLastError()); + } + break; + default: + break; + } +} + +/* + * A variable named _tls_used contains the TLS directory, which contains a list + * of pointers to callback functions. Normally, the linker won't retain this + * variable unless the executable has implicit thread-local variables, defined + * using the __declspec(thread) extended storage-class modifier. libhdfs + * doesn't use __declspec(thread), and we have no guarantee that the executable + * linked to libhdfs will use __declspec(thread). By forcing the linker to + * reference _tls_used, we guarantee that the binary retains the TLS directory. + * See Microsoft Visual Studio 10.0/VC/crt/src/tlssup.c . + */ +#pragma comment(linker, "/INCLUDE:_tls_used") + +/* + * We must retain a pointer to the callback function. Force the linker to keep + * this symbol, even though it appears that nothing in our source code uses it. + */ +#pragma comment(linker, "/INCLUDE:pTlsCallback") + +/* + * Define constant pointer to our callback, and tell the linker to pin it into + * the TLS directory so that it receives thread callbacks. Use external linkage + * to protect against the linker discarding the seemingly unused symbol. + */ +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK pTlsCallback; +const PIMAGE_TLS_CALLBACK pTlsCallback = tlsCallback; +#pragma const_seg() + +int threadLocalStorageGet(JNIEnv **env) +{ + LPVOID tls; + DWORD ret; + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + gTlsIndex = TlsAlloc(); + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + fprintf(stderr, + "threadLocalStorageGet: TlsAlloc failed with error %d\n", + TLS_OUT_OF_INDEXES); + return TLS_OUT_OF_INDEXES; + } + } + tls = TlsGetValue(gTlsIndex); + if (tls) { + *env = tls; + return 0; + } else { + ret = GetLastError(); + if (ERROR_SUCCESS == ret) { + /* Thread-local storage contains NULL, because we haven't set it yet. */ + *env = NULL; + return 0; + } else { + /* + * The API call failed. According to documentation, TlsGetValue cannot + * fail as long as the index is a valid index from a successful TlsAlloc + * call. This error handling is purely defensive. + */ + fprintf(stderr, + "threadLocalStorageGet: TlsGetValue failed with error %d\n", ret); + return ret; + } + } +} + +int threadLocalStorageSet(JNIEnv *env) +{ + DWORD ret = 0; + if (!TlsSetValue(gTlsIndex, (LPVOID)env)) { + ret = GetLastError(); + fprintf(stderr, + "threadLocalStorageSet: TlsSetValue failed with error %d\n", + ret); + detachCurrentThreadFromJvm(env); + } + return ret; +} diff --git a/libhdfs/hdfs_2_6/os/windows/unistd.h b/libhdfs/hdfs_2_6/os/windows/unistd.h new file mode 100644 index 0000000..b82ce48 --- /dev/null +++ b/libhdfs/hdfs_2_6/os/windows/unistd.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_UNISTD_H +#define LIBHDFS_UNISTD_H + +/* On Windows, unistd.h does not exist, so manually define what we need. */ + +#include /* Declares getpid(). */ +#include + +/* Re-route sleep to Sleep, converting units from seconds to milliseconds. */ +#define sleep(seconds) Sleep((seconds) * 1000) +#endif diff --git a/libhdfs/hdfs_2_7/common/htable.c b/libhdfs/hdfs_2_7/common/htable.c new file mode 100644 index 0000000..50c89ea --- /dev/null +++ b/libhdfs/hdfs_2_7/common/htable.c @@ -0,0 +1,287 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/htable.h" + +#include +#include +#include +#include +#include + +struct htable_pair { + void *key; + void *val; +}; + +/** + * A hash table which uses linear probing. + */ +struct htable { + uint32_t capacity; + uint32_t used; + htable_hash_fn_t hash_fun; + htable_eq_fn_t eq_fun; + struct htable_pair *elem; +}; + +/** + * An internal function for inserting a value into the hash table. + * + * Note: this function assumes that you have made enough space in the table. + * + * @param nelem The new element to insert. + * @param capacity The capacity of the hash table. + * @param hash_fun The hash function to use. + * @param key The key to insert. + * @param val The value to insert. + */ +static void htable_insert_internal(struct htable_pair *nelem, + uint32_t capacity, htable_hash_fn_t hash_fun, void *key, + void *val) +{ + uint32_t i; + + i = hash_fun(key, capacity); + while (1) { + if (!nelem[i].key) { + nelem[i].key = key; + nelem[i].val = val; + return; + } + i++; + if (i == capacity) { + i = 0; + } + } +} + +static int htable_realloc(struct htable *htable, uint32_t new_capacity) +{ + struct htable_pair *nelem; + uint32_t i, old_capacity = htable->capacity; + htable_hash_fn_t hash_fun = htable->hash_fun; + + nelem = calloc(new_capacity, sizeof(struct htable_pair)); + if (!nelem) { + return ENOMEM; + } + for (i = 0; i < old_capacity; i++) { + struct htable_pair *pair = htable->elem + i; + if (pair->key) { + htable_insert_internal(nelem, new_capacity, hash_fun, + pair->key, pair->val); + } + } + free(htable->elem); + htable->elem = nelem; + htable->capacity = new_capacity; + return 0; +} + +static uint32_t round_up_to_power_of_2(uint32_t i) +{ + if (i == 0) { + return 1; + } + i--; + i |= i >> 1; + i |= i >> 2; + i |= i >> 4; + i |= i >> 8; + i |= i >> 16; + i++; + return i; +} + +struct htable *htable_alloc(uint32_t size, + htable_hash_fn_t hash_fun, htable_eq_fn_t eq_fun) +{ + struct htable *htable; + + htable = calloc(1, sizeof(*htable)); + if (!htable) { + return NULL; + } + size = round_up_to_power_of_2(size); + if (size < HTABLE_MIN_SIZE) { + size = HTABLE_MIN_SIZE; + } + htable->hash_fun = hash_fun; + htable->eq_fun = eq_fun; + htable->used = 0; + if (htable_realloc(htable, size)) { + free(htable); + return NULL; + } + return htable; +} + +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx) +{ + uint32_t i; + + for (i = 0; i != htable->capacity; ++i) { + struct htable_pair *elem = htable->elem + i; + if (elem->key) { + fun(ctx, elem->key, elem->val); + } + } +} + +void htable_free(struct htable *htable) +{ + if (htable) { + free(htable->elem); + free(htable); + } +} + +int htable_put(struct htable *htable, void *key, void *val) +{ + int ret; + uint32_t nused; + + // NULL is not a valid key value. + // This helps us implement htable_get_internal efficiently, since we know + // that we can stop when we encounter the first NULL key. + if (!key) { + return EINVAL; + } + // NULL is not a valid value. Otherwise the results of htable_get would + // be confusing (does a NULL return mean entry not found, or that the + // entry was found and was NULL?) + if (!val) { + return EINVAL; + } + // Re-hash if we have used more than half of the hash table + nused = htable->used + 1; + if (nused >= (htable->capacity / 2)) { + ret = htable_realloc(htable, htable->capacity * 2); + if (ret) + return ret; + } + htable_insert_internal(htable->elem, htable->capacity, + htable->hash_fun, key, val); + htable->used++; + return 0; +} + +static int htable_get_internal(const struct htable *htable, + const void *key, uint32_t *out) +{ + uint32_t start_idx, idx; + + start_idx = htable->hash_fun(key, htable->capacity); + idx = start_idx; + while (1) { + struct htable_pair *pair = htable->elem + idx; + if (!pair->key) { + // We always maintain the invariant that the entries corresponding + // to a given key are stored in a contiguous block, not separated + // by any NULLs. So if we encounter a NULL, our search is over. + return ENOENT; + } else if (htable->eq_fun(pair->key, key)) { + *out = idx; + return 0; + } + idx++; + if (idx == htable->capacity) { + idx = 0; + } + if (idx == start_idx) { + return ENOENT; + } + } +} + +void *htable_get(const struct htable *htable, const void *key) +{ + uint32_t idx; + + if (htable_get_internal(htable, key, &idx)) { + return NULL; + } + return htable->elem[idx].val; +} + +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val) +{ + uint32_t hole, i; + const void *nkey; + + if (htable_get_internal(htable, key, &hole)) { + *found_key = NULL; + *found_val = NULL; + return; + } + i = hole; + htable->used--; + // We need to maintain the compactness invariant used in + // htable_get_internal. This invariant specifies that the entries for any + // given key are never separated by NULLs (although they may be separated + // by entries for other keys.) + while (1) { + i++; + if (i == htable->capacity) { + i = 0; + } + nkey = htable->elem[i].key; + if (!nkey) { + *found_key = htable->elem[hole].key; + *found_val = htable->elem[hole].val; + htable->elem[hole].key = NULL; + htable->elem[hole].val = NULL; + return; + } else if (htable->eq_fun(key, nkey)) { + htable->elem[hole].key = htable->elem[i].key; + htable->elem[hole].val = htable->elem[i].val; + hole = i; + } + } +} + +uint32_t htable_used(const struct htable *htable) +{ + return htable->used; +} + +uint32_t htable_capacity(const struct htable *htable) +{ + return htable->capacity; +} + +uint32_t ht_hash_string(const void *str, uint32_t max) +{ + const char *s = str; + uint32_t hash = 0; + + while (*s) { + hash = (hash * 31) + *s; + s++; + } + return hash % max; +} + +int ht_compare_string(const void *a, const void *b) +{ + return strcmp(a, b) == 0; +} + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_2_7/common/htable.h b/libhdfs/hdfs_2_7/common/htable.h new file mode 100644 index 0000000..33f1229 --- /dev/null +++ b/libhdfs/hdfs_2_7/common/htable.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HADOOP_CORE_COMMON_HASH_TABLE +#define HADOOP_CORE_COMMON_HASH_TABLE + +#include +#include +#include + +#define HTABLE_MIN_SIZE 4 + +struct htable; + +/** + * An HTable hash function. + * + * @param key The key. + * @param capacity The total capacity. + * + * @return The hash slot. Must be less than the capacity. + */ +typedef uint32_t (*htable_hash_fn_t)(const void *key, uint32_t capacity); + +/** + * An HTable equality function. Compares two keys. + * + * @param a First key. + * @param b Second key. + * + * @return nonzero if the keys are equal. + */ +typedef int (*htable_eq_fn_t)(const void *a, const void *b); + +/** + * Allocate a new hash table. + * + * @param capacity The minimum suggested starting capacity. + * @param hash_fun The hash function to use in this hash table. + * @param eq_fun The equals function to use in this hash table. + * + * @return The new hash table on success; NULL on OOM. + */ +struct htable *htable_alloc(uint32_t capacity, htable_hash_fn_t hash_fun, + htable_eq_fn_t eq_fun); + +typedef void (*visitor_fn_t)(void *ctx, void *key, void *val); + +/** + * Visit all of the entries in the hash table. + * + * @param htable The hash table. + * @param fun The callback function to invoke on each key and value. + * @param ctx Context pointer to pass to the callback. + */ +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx); + +/** + * Free the hash table. + * + * It is up the calling code to ensure that the keys and values inside the + * table are de-allocated, if that is necessary. + * + * @param htable The hash table. + */ +void htable_free(struct htable *htable); + +/** + * Add an entry to the hash table. + * + * @param htable The hash table. + * @param key The key to add. This cannot be NULL. + * @param fun The value to add. This cannot be NULL. + * + * @return 0 on success; + * EEXIST if the value already exists in the table; + * ENOMEM if there is not enough memory to add the element. + * EFBIG if the hash table has too many entries to fit in 32 + * bits. + */ +int htable_put(struct htable *htable, void *key, void *val); + +/** + * Get an entry from the hash table. + * + * @param htable The hash table. + * @param key The key to find. + * + * @return NULL if there is no such entry; the entry otherwise. + */ +void *htable_get(const struct htable *htable, const void *key); + +/** + * Get an entry from the hash table and remove it. + * + * @param htable The hash table. + * @param key The key for the entry find and remove. + * @param found_key (out param) NULL if the entry was not found; the found key + * otherwise. + * @param found_val (out param) NULL if the entry was not found; the found + * value otherwise. + */ +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val); + +/** + * Get the number of entries used in the hash table. + * + * @param htable The hash table. + * + * @return The number of entries used in the hash table. + */ +uint32_t htable_used(const struct htable *htable); + +/** + * Get the capacity of the hash table. + * + * @param htable The hash table. + * + * @return The capacity of the hash table. + */ +uint32_t htable_capacity(const struct htable *htable); + +/** + * Hash a string. + * + * @param str The string. + * @param max Maximum hash value + * + * @return A number less than max. + */ +uint32_t ht_hash_string(const void *str, uint32_t max); + +/** + * Compare two strings. + * + * @param a The first string. + * @param b The second string. + * + * @return 1 if the strings are identical; 0 otherwise. + */ +int ht_compare_string(const void *a, const void *b); + +#endif + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_2_7/exception.c b/libhdfs/hdfs_2_7/exception.c new file mode 100644 index 0000000..eb7115c --- /dev/null +++ b/libhdfs/hdfs_2_7/exception.c @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "java.lang.UnsupportedOperationException", + 0, + ENOTSUP, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jstring jStr = NULL; + jvalue jVal; + jthrowable jthr; + const char *stackTrace; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "ExceptionUtils::getStackTrace error.)\n", className); + destroyLocalReference(env, jthr); + } else { + jStr = jVal.l; + stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL); + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "GetStringUTFChars error.)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + (*env)->ReleaseStringUTFChars(env, jStr, stackTrace); + } + } + } + destroyLocalReference(env, jStr); + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_2_7/exception.h b/libhdfs/hdfs_2_7/exception.h new file mode 100644 index 0000000..5fa7fa6 --- /dev/null +++ b/libhdfs/hdfs_2_7/exception.h @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * PrintExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Print out information about the pending exception and free it. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/libhdfs/hdfs_2_7/hdfs.c b/libhdfs/hdfs_2_7/hdfs.c new file mode 100644 index 0000000..5c39dde --- /dev/null +++ b/libhdfs/hdfs_2_7/hdfs.c @@ -0,0 +1,3341 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +int hdfsFileClearReadStatistics(hdfsFile file) +{ + jthrowable jthr; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return EINTERNAL; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "clearReadStatistics", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileClearReadStatistics: clearReadStatistics failed"); + goto done; + } + ret = 0; +done: + if (ret) { + errno = ret; + return ret; + } + return 0; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) +{ + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + jobject jPath = NULL; + + JNIEnv *env = getJNIEnv(); + + if (!env) { + errno = EINTERNAL; + return -1; + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): constructNewObjectOfPath", path); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "truncate", JMETHOD2(JPARAM(HADOOP_PATH), "J", "Z"), + jPath, newlength); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): FileSystem#truncate", path); + return -1; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +int hdfsUnbufferFile(hdfsFile file) +{ + int ret; + jthrowable jthr; + JNIEnv *env = getJNIEnv(); + + if (!env) { + ret = EINTERNAL; + goto done; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = ENOTSUP; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, HADOOP_ISTRM, + "unbuffer", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + HADOOP_ISTRM "#unbuffer failed:"); + goto done; + } + ret = 0; + +done: + errno = ret; + return ret; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + const char *interface; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + interface = (file->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + const char *interface; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + interface = (f->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + + + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_2.7.h b/libhdfs/hdfs_2_7/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_2.7.h rename to libhdfs/hdfs_2_7/hdfs.h diff --git a/libhdfs/hdfs_2_7/jni_helper.c b/libhdfs/hdfs_2_7/jni_helper.c new file mode 100644 index 0000000..50d9681 --- /dev/null +++ b/libhdfs/hdfs_2_7/jni_helper.c @@ -0,0 +1,595 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" +#include "platform.h" +#include "common/htable.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include + +static struct htable *gClassRefHTable = NULL; + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + jmethodID mid = 0; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jthrowable jthr = NULL; + jclass local_clazz = NULL; + jclass clazz = NULL; + int ret; + + mutexLock(&hdfsHashMutex); + if (!gClassRefHTable) { + gClassRefHTable = htable_alloc(MAX_HASH_TABLE_ELEM, ht_hash_string, + ht_compare_string); + if (!gClassRefHTable) { + jthr = newRuntimeError(env, "htable_alloc failed\n"); + goto done; + } + } + clazz = htable_get(gClassRefHTable, className); + if (clazz) { + *out = clazz; + goto done; + } + local_clazz = (*env)->FindClass(env,className); + if (!local_clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clazz = (*env)->NewGlobalRef(env, local_clazz); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + ret = htable_put(gClassRefHTable, (void*)className, clazz); + if (ret) { + jthr = newRuntimeError(env, "htable_put failed with error " + "code %d\n", ret); + goto done; + } + *out = clazz; + jthr = NULL; +done: + mutexUnlock(&hdfsHashMutex); + (*env)->DeleteLocalRef(env, local_clazz); + if (jthr && clazz) { + (*env)->DeleteGlobalRef(env, clazz); + } + return jthr; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + JNIEnv *env; + THREAD_LOCAL_STORAGE_GET_QUICK(); + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&env)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (env) { + mutexUnlock(&jvmMutex); + return env; + } + + env = getGlobalJNIEnv(); + mutexUnlock(&jvmMutex); + if (!env) { + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + return NULL; + } + if (threadLocalStorageSet(env)) { + return NULL; + } + THREAD_LOCAL_STORAGE_SET_QUICK(env); + return env; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_2_7/jni_helper.h b/libhdfs/hdfs_2_7/jni_helper.h new file mode 100644 index 0000000..90accc7 --- /dev/null +++ b/libhdfs/hdfs_2_7/jni_helper.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_2_7/os/mutexes.h b/libhdfs/hdfs_2_7/os/mutexes.h new file mode 100644 index 0000000..da30bf4 --- /dev/null +++ b/libhdfs/hdfs_2_7/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting the class reference hash table. */ +extern mutex hdfsHashMutex; + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/libhdfs/hdfs_2_7/os/posix/mutexes.c b/libhdfs/hdfs_2_7/os/posix/mutexes.c new file mode 100644 index 0000000..c4c2f26 --- /dev/null +++ b/libhdfs/hdfs_2_7/os/posix/mutexes.c @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +mutex jvmMutex = PTHREAD_MUTEX_INITIALIZER; + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/libhdfs/hdfs_2_7/os/posix/platform.h b/libhdfs/hdfs_2_7/os/posix/platform.h new file mode 100644 index 0000000..c63bbf9 --- /dev/null +++ b/libhdfs/hdfs_2_7/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/libhdfs/hdfs_2_7/os/posix/thread.c b/libhdfs/hdfs_2_7/os/posix/thread.c new file mode 100644 index 0000000..af0c61f --- /dev/null +++ b/libhdfs/hdfs_2_7/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/libhdfs/hdfs_2_7/os/posix/thread_local_storage.c b/libhdfs/hdfs_2_7/os/posix/thread_local_storage.c new file mode 100644 index 0000000..2f70e2c --- /dev/null +++ b/libhdfs/hdfs_2_7/os/posix/thread_local_storage.c @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +static void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + JNIEnv *env = v; + jint ret; + + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } +} + +int threadLocalStorageGet(JNIEnv **env) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *env = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(JNIEnv *env) +{ + int ret = pthread_setspecific(gTlsKey, env); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(env); + } + return ret; +} diff --git a/libhdfs/hdfs_2_7/os/thread.h b/libhdfs/hdfs_2_7/os/thread.h new file mode 100644 index 0000000..ae425d3 --- /dev/null +++ b/libhdfs/hdfs_2_7/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/libhdfs/hdfs_2_7/os/thread_local_storage.h b/libhdfs/hdfs_2_7/os/thread_local_storage.h new file mode 100644 index 0000000..a40d567 --- /dev/null +++ b/libhdfs/hdfs_2_7/os/thread_local_storage.h @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK() \ + static __thread JNIEnv *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + return quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(env) \ + { \ + quickTlsEnv = (env); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK() + #define THREAD_LOCAL_STORAGE_SET_QUICK(env) +#endif + +/** + * Gets the JNIEnv in thread-local storage for the current thread. If the call + * succeeds, and there is a JNIEnv associated with this thread, then returns 0 + * and populates env. If the call succeeds, but there is no JNIEnv associated + * with this thread, then returns 0 and sets JNIEnv to NULL. If the call fails, + * then returns non-zero. Only one thread at a time may execute this function. + * The caller is responsible for enforcing mutual exclusion. + * + * @param env JNIEnv out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(JNIEnv **env); + +/** + * Sets the JNIEnv in thread-local storage for the current thread. + * + * @param env JNIEnv to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(JNIEnv *env); + +#endif diff --git a/libhdfs/hdfs_2_7/os/windows/inttypes.h b/libhdfs/hdfs_2_7/os/windows/inttypes.h new file mode 100644 index 0000000..a520d15 --- /dev/null +++ b/libhdfs/hdfs_2_7/os/windows/inttypes.h @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_INTTYPES_H +#define LIBHDFS_INTTYPES_H + +/* On Windows, inttypes.h does not exist, so manually define what we need. */ + +#define PRId64 "I64d" +#define PRIu64 "I64u" +typedef unsigned __int64 uint64_t; + +#endif diff --git a/libhdfs/hdfs_2_7/os/windows/mutexes.c b/libhdfs/hdfs_2_7/os/windows/mutexes.c new file mode 100644 index 0000000..875f033 --- /dev/null +++ b/libhdfs/hdfs_2_7/os/windows/mutexes.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include + +mutex hdfsHashMutex; +mutex jvmMutex; + +/** + * Unfortunately, there is no simple static initializer for a critical section. + * Instead, the API requires calling InitializeCriticalSection. Since libhdfs + * lacks an explicit initialization function, there is no obvious existing place + * for the InitializeCriticalSection calls. To work around this, we define an + * initialization function and instruct the linker to set a pointer to that + * function as a user-defined global initializer. See discussion of CRT + * Initialization: + * http://msdn.microsoft.com/en-us/library/bb918180.aspx + */ +static void __cdecl initializeMutexes(void) { + InitializeCriticalSection(&hdfsHashMutex); + InitializeCriticalSection(&jvmMutex); +} +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) +const void (__cdecl *pInitialize)(void) = initializeMutexes; + +int mutexLock(mutex *m) { + EnterCriticalSection(m); + return 0; +} + +int mutexUnlock(mutex *m) { + LeaveCriticalSection(m); + return 0; +} diff --git a/libhdfs/hdfs_2_7/os/windows/platform.h b/libhdfs/hdfs_2_7/os/windows/platform.h new file mode 100644 index 0000000..9eedfde --- /dev/null +++ b/libhdfs/hdfs_2_7/os/windows/platform.h @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include +#include +#include + +/* + * O_ACCMODE defined to match Linux definition. + */ +#ifndef O_ACCMODE +#define O_ACCMODE 0x0003 +#endif + +/* + * Windows has a different name for its maximum path length constant. + */ +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +/* + * Windows does not define EDQUOT and ESTALE in errno.h. The closest equivalents + * are these constants from winsock.h. + */ +#ifndef EDQUOT +#define EDQUOT WSAEDQUOT +#endif + +#ifndef ESTALE +#define ESTALE WSAESTALE +#endif + +/* + * gcc-style type-checked format arguments are not supported on Windows, so just + * stub this macro. + */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) + +/* + * Define macros for various string formatting functions not defined on Windows. + * Where possible, we reroute to one of the secure CRT variants. On Windows, + * the preprocessor does support variadic macros, even though they weren't + * defined until C99. + */ +#define snprintf(str, size, format, ...) \ + _snprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) +#define strncpy(dest, src, n) \ + strncpy_s((dest), (n), (src), _TRUNCATE) +#define strtok_r(str, delim, saveptr) \ + strtok_s((str), (delim), (saveptr)) +#define vsnprintf(str, size, format, ...) \ + vsnprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) + +/* + * Mutex data type defined as Windows CRITICAL_SECTION. A critical section (not + * Windows mutex) is used, because libhdfs only needs synchronization of multiple + * threads within a single process, not synchronization across process + * boundaries. + */ +typedef CRITICAL_SECTION mutex; + +/* + * Thread data type defined as HANDLE to a Windows thread. + */ +typedef HANDLE threadId; + +#endif diff --git a/libhdfs/hdfs_2_7/os/windows/thread.c b/libhdfs/hdfs_2_7/os/windows/thread.c new file mode 100644 index 0000000..f5cc2a7 --- /dev/null +++ b/libhdfs/hdfs_2_7/os/windows/thread.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by CreateThread. + * + * @param toRun thread to run + * @return DWORD result of running thread (always 0) + */ +static DWORD WINAPI runThread(LPVOID toRun) { + const thread *t = toRun; + t->start(t->arg); + return 0; +} + +int threadCreate(thread *t) { + DWORD ret = 0; + HANDLE h; + h = CreateThread(NULL, 0, runThread, t, 0, NULL); + if (h) { + t->id = h; + } else { + ret = GetLastError(); + fprintf(stderr, "threadCreate: CreateThread failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + DWORD ret = WaitForSingleObject(t->id, INFINITE); + switch (ret) { + case WAIT_OBJECT_0: + break; + case WAIT_FAILED: + ret = GetLastError(); + fprintf(stderr, "threadJoin: WaitForSingleObject failed with error %d\n", + ret); + break; + default: + fprintf(stderr, "threadJoin: WaitForSingleObject unexpected error %d\n", + ret); + break; + } + return ret; +} diff --git a/libhdfs/hdfs_2_7/os/windows/thread_local_storage.c b/libhdfs/hdfs_2_7/os/windows/thread_local_storage.c new file mode 100644 index 0000000..4c415e1 --- /dev/null +++ b/libhdfs/hdfs_2_7/os/windows/thread_local_storage.c @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static DWORD gTlsIndex = TLS_OUT_OF_INDEXES; + +/** + * If the current thread has a JNIEnv in thread-local storage, then detaches the + * current thread from the JVM. + */ +static void detachCurrentThreadFromJvm() +{ + JNIEnv *env = NULL; + JavaVM *vm; + jint ret; + if (threadLocalStorageGet(&env) || !env) { + return; + } + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, + "detachCurrentThreadFromJvm: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } +} + +/** + * Unlike pthreads, the Windows API does not seem to provide a convenient way to + * hook a callback onto thread shutdown. However, the Windows portable + * executable format does define a concept of thread-local storage callbacks. + * Here, we define a function and instruct the linker to set a pointer to that + * function in the segment for thread-local storage callbacks. See page 85 of + * Microsoft Portable Executable and Common Object File Format Specification: + * http://msdn.microsoft.com/en-us/gg463119.aspx + * This technique only works for implicit linking (OS loads DLL on demand), not + * for explicit linking (user code calls LoadLibrary directly). This effectively + * means that we have a known limitation: libhdfs may not work correctly if a + * Windows application attempts to use it via explicit linking. + * + * @param h module handle + * @param reason the reason for calling the callback + * @param pv reserved, unused + */ +static void NTAPI tlsCallback(PVOID h, DWORD reason, PVOID pv) +{ + DWORD tlsIndex; + switch (reason) { + case DLL_THREAD_DETACH: + detachCurrentThreadFromJvm(); + break; + case DLL_PROCESS_DETACH: + detachCurrentThreadFromJvm(); + tlsIndex = gTlsIndex; + gTlsIndex = TLS_OUT_OF_INDEXES; + if (!TlsFree(tlsIndex)) { + fprintf(stderr, "tlsCallback: TlsFree failed with error %d\n", + GetLastError()); + } + break; + default: + break; + } +} + +/* + * A variable named _tls_used contains the TLS directory, which contains a list + * of pointers to callback functions. Normally, the linker won't retain this + * variable unless the executable has implicit thread-local variables, defined + * using the __declspec(thread) extended storage-class modifier. libhdfs + * doesn't use __declspec(thread), and we have no guarantee that the executable + * linked to libhdfs will use __declspec(thread). By forcing the linker to + * reference _tls_used, we guarantee that the binary retains the TLS directory. + * See Microsoft Visual Studio 10.0/VC/crt/src/tlssup.c . + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#endif + +/* + * We must retain a pointer to the callback function. Force the linker to keep + * this symbol, even though it appears that nothing in our source code uses it. + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:pTlsCallback") +#else +#pragma comment(linker, "/INCLUDE:_pTlsCallback") +#endif + +/* + * Define constant pointer to our callback, and tell the linker to pin it into + * the TLS directory so that it receives thread callbacks. Use external linkage + * to protect against the linker discarding the seemingly unused symbol. + */ +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK pTlsCallback; +const PIMAGE_TLS_CALLBACK pTlsCallback = tlsCallback; +#pragma const_seg() + +int threadLocalStorageGet(JNIEnv **env) +{ + LPVOID tls; + DWORD ret; + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + gTlsIndex = TlsAlloc(); + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + fprintf(stderr, + "threadLocalStorageGet: TlsAlloc failed with error %d\n", + TLS_OUT_OF_INDEXES); + return TLS_OUT_OF_INDEXES; + } + } + tls = TlsGetValue(gTlsIndex); + if (tls) { + *env = tls; + return 0; + } else { + ret = GetLastError(); + if (ERROR_SUCCESS == ret) { + /* Thread-local storage contains NULL, because we haven't set it yet. */ + *env = NULL; + return 0; + } else { + /* + * The API call failed. According to documentation, TlsGetValue cannot + * fail as long as the index is a valid index from a successful TlsAlloc + * call. This error handling is purely defensive. + */ + fprintf(stderr, + "threadLocalStorageGet: TlsGetValue failed with error %d\n", ret); + return ret; + } + } +} + +int threadLocalStorageSet(JNIEnv *env) +{ + DWORD ret = 0; + if (!TlsSetValue(gTlsIndex, (LPVOID)env)) { + ret = GetLastError(); + fprintf(stderr, + "threadLocalStorageSet: TlsSetValue failed with error %d\n", + ret); + detachCurrentThreadFromJvm(env); + } + return ret; +} diff --git a/libhdfs/hdfs_2_7/os/windows/unistd.h b/libhdfs/hdfs_2_7/os/windows/unistd.h new file mode 100644 index 0000000..b82ce48 --- /dev/null +++ b/libhdfs/hdfs_2_7/os/windows/unistd.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_UNISTD_H +#define LIBHDFS_UNISTD_H + +/* On Windows, unistd.h does not exist, so manually define what we need. */ + +#include /* Declares getpid(). */ +#include + +/* Re-route sleep to Sleep, converting units from seconds to milliseconds. */ +#define sleep(seconds) Sleep((seconds) * 1000) +#endif diff --git a/libhdfs/hdfs_2_8/common/htable.c b/libhdfs/hdfs_2_8/common/htable.c new file mode 100644 index 0000000..50c89ea --- /dev/null +++ b/libhdfs/hdfs_2_8/common/htable.c @@ -0,0 +1,287 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/htable.h" + +#include +#include +#include +#include +#include + +struct htable_pair { + void *key; + void *val; +}; + +/** + * A hash table which uses linear probing. + */ +struct htable { + uint32_t capacity; + uint32_t used; + htable_hash_fn_t hash_fun; + htable_eq_fn_t eq_fun; + struct htable_pair *elem; +}; + +/** + * An internal function for inserting a value into the hash table. + * + * Note: this function assumes that you have made enough space in the table. + * + * @param nelem The new element to insert. + * @param capacity The capacity of the hash table. + * @param hash_fun The hash function to use. + * @param key The key to insert. + * @param val The value to insert. + */ +static void htable_insert_internal(struct htable_pair *nelem, + uint32_t capacity, htable_hash_fn_t hash_fun, void *key, + void *val) +{ + uint32_t i; + + i = hash_fun(key, capacity); + while (1) { + if (!nelem[i].key) { + nelem[i].key = key; + nelem[i].val = val; + return; + } + i++; + if (i == capacity) { + i = 0; + } + } +} + +static int htable_realloc(struct htable *htable, uint32_t new_capacity) +{ + struct htable_pair *nelem; + uint32_t i, old_capacity = htable->capacity; + htable_hash_fn_t hash_fun = htable->hash_fun; + + nelem = calloc(new_capacity, sizeof(struct htable_pair)); + if (!nelem) { + return ENOMEM; + } + for (i = 0; i < old_capacity; i++) { + struct htable_pair *pair = htable->elem + i; + if (pair->key) { + htable_insert_internal(nelem, new_capacity, hash_fun, + pair->key, pair->val); + } + } + free(htable->elem); + htable->elem = nelem; + htable->capacity = new_capacity; + return 0; +} + +static uint32_t round_up_to_power_of_2(uint32_t i) +{ + if (i == 0) { + return 1; + } + i--; + i |= i >> 1; + i |= i >> 2; + i |= i >> 4; + i |= i >> 8; + i |= i >> 16; + i++; + return i; +} + +struct htable *htable_alloc(uint32_t size, + htable_hash_fn_t hash_fun, htable_eq_fn_t eq_fun) +{ + struct htable *htable; + + htable = calloc(1, sizeof(*htable)); + if (!htable) { + return NULL; + } + size = round_up_to_power_of_2(size); + if (size < HTABLE_MIN_SIZE) { + size = HTABLE_MIN_SIZE; + } + htable->hash_fun = hash_fun; + htable->eq_fun = eq_fun; + htable->used = 0; + if (htable_realloc(htable, size)) { + free(htable); + return NULL; + } + return htable; +} + +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx) +{ + uint32_t i; + + for (i = 0; i != htable->capacity; ++i) { + struct htable_pair *elem = htable->elem + i; + if (elem->key) { + fun(ctx, elem->key, elem->val); + } + } +} + +void htable_free(struct htable *htable) +{ + if (htable) { + free(htable->elem); + free(htable); + } +} + +int htable_put(struct htable *htable, void *key, void *val) +{ + int ret; + uint32_t nused; + + // NULL is not a valid key value. + // This helps us implement htable_get_internal efficiently, since we know + // that we can stop when we encounter the first NULL key. + if (!key) { + return EINVAL; + } + // NULL is not a valid value. Otherwise the results of htable_get would + // be confusing (does a NULL return mean entry not found, or that the + // entry was found and was NULL?) + if (!val) { + return EINVAL; + } + // Re-hash if we have used more than half of the hash table + nused = htable->used + 1; + if (nused >= (htable->capacity / 2)) { + ret = htable_realloc(htable, htable->capacity * 2); + if (ret) + return ret; + } + htable_insert_internal(htable->elem, htable->capacity, + htable->hash_fun, key, val); + htable->used++; + return 0; +} + +static int htable_get_internal(const struct htable *htable, + const void *key, uint32_t *out) +{ + uint32_t start_idx, idx; + + start_idx = htable->hash_fun(key, htable->capacity); + idx = start_idx; + while (1) { + struct htable_pair *pair = htable->elem + idx; + if (!pair->key) { + // We always maintain the invariant that the entries corresponding + // to a given key are stored in a contiguous block, not separated + // by any NULLs. So if we encounter a NULL, our search is over. + return ENOENT; + } else if (htable->eq_fun(pair->key, key)) { + *out = idx; + return 0; + } + idx++; + if (idx == htable->capacity) { + idx = 0; + } + if (idx == start_idx) { + return ENOENT; + } + } +} + +void *htable_get(const struct htable *htable, const void *key) +{ + uint32_t idx; + + if (htable_get_internal(htable, key, &idx)) { + return NULL; + } + return htable->elem[idx].val; +} + +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val) +{ + uint32_t hole, i; + const void *nkey; + + if (htable_get_internal(htable, key, &hole)) { + *found_key = NULL; + *found_val = NULL; + return; + } + i = hole; + htable->used--; + // We need to maintain the compactness invariant used in + // htable_get_internal. This invariant specifies that the entries for any + // given key are never separated by NULLs (although they may be separated + // by entries for other keys.) + while (1) { + i++; + if (i == htable->capacity) { + i = 0; + } + nkey = htable->elem[i].key; + if (!nkey) { + *found_key = htable->elem[hole].key; + *found_val = htable->elem[hole].val; + htable->elem[hole].key = NULL; + htable->elem[hole].val = NULL; + return; + } else if (htable->eq_fun(key, nkey)) { + htable->elem[hole].key = htable->elem[i].key; + htable->elem[hole].val = htable->elem[i].val; + hole = i; + } + } +} + +uint32_t htable_used(const struct htable *htable) +{ + return htable->used; +} + +uint32_t htable_capacity(const struct htable *htable) +{ + return htable->capacity; +} + +uint32_t ht_hash_string(const void *str, uint32_t max) +{ + const char *s = str; + uint32_t hash = 0; + + while (*s) { + hash = (hash * 31) + *s; + s++; + } + return hash % max; +} + +int ht_compare_string(const void *a, const void *b) +{ + return strcmp(a, b) == 0; +} + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_2_8/common/htable.h b/libhdfs/hdfs_2_8/common/htable.h new file mode 100644 index 0000000..33f1229 --- /dev/null +++ b/libhdfs/hdfs_2_8/common/htable.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HADOOP_CORE_COMMON_HASH_TABLE +#define HADOOP_CORE_COMMON_HASH_TABLE + +#include +#include +#include + +#define HTABLE_MIN_SIZE 4 + +struct htable; + +/** + * An HTable hash function. + * + * @param key The key. + * @param capacity The total capacity. + * + * @return The hash slot. Must be less than the capacity. + */ +typedef uint32_t (*htable_hash_fn_t)(const void *key, uint32_t capacity); + +/** + * An HTable equality function. Compares two keys. + * + * @param a First key. + * @param b Second key. + * + * @return nonzero if the keys are equal. + */ +typedef int (*htable_eq_fn_t)(const void *a, const void *b); + +/** + * Allocate a new hash table. + * + * @param capacity The minimum suggested starting capacity. + * @param hash_fun The hash function to use in this hash table. + * @param eq_fun The equals function to use in this hash table. + * + * @return The new hash table on success; NULL on OOM. + */ +struct htable *htable_alloc(uint32_t capacity, htable_hash_fn_t hash_fun, + htable_eq_fn_t eq_fun); + +typedef void (*visitor_fn_t)(void *ctx, void *key, void *val); + +/** + * Visit all of the entries in the hash table. + * + * @param htable The hash table. + * @param fun The callback function to invoke on each key and value. + * @param ctx Context pointer to pass to the callback. + */ +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx); + +/** + * Free the hash table. + * + * It is up the calling code to ensure that the keys and values inside the + * table are de-allocated, if that is necessary. + * + * @param htable The hash table. + */ +void htable_free(struct htable *htable); + +/** + * Add an entry to the hash table. + * + * @param htable The hash table. + * @param key The key to add. This cannot be NULL. + * @param fun The value to add. This cannot be NULL. + * + * @return 0 on success; + * EEXIST if the value already exists in the table; + * ENOMEM if there is not enough memory to add the element. + * EFBIG if the hash table has too many entries to fit in 32 + * bits. + */ +int htable_put(struct htable *htable, void *key, void *val); + +/** + * Get an entry from the hash table. + * + * @param htable The hash table. + * @param key The key to find. + * + * @return NULL if there is no such entry; the entry otherwise. + */ +void *htable_get(const struct htable *htable, const void *key); + +/** + * Get an entry from the hash table and remove it. + * + * @param htable The hash table. + * @param key The key for the entry find and remove. + * @param found_key (out param) NULL if the entry was not found; the found key + * otherwise. + * @param found_val (out param) NULL if the entry was not found; the found + * value otherwise. + */ +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val); + +/** + * Get the number of entries used in the hash table. + * + * @param htable The hash table. + * + * @return The number of entries used in the hash table. + */ +uint32_t htable_used(const struct htable *htable); + +/** + * Get the capacity of the hash table. + * + * @param htable The hash table. + * + * @return The capacity of the hash table. + */ +uint32_t htable_capacity(const struct htable *htable); + +/** + * Hash a string. + * + * @param str The string. + * @param max Maximum hash value + * + * @return A number less than max. + */ +uint32_t ht_hash_string(const void *str, uint32_t max); + +/** + * Compare two strings. + * + * @param a The first string. + * @param b The second string. + * + * @return 1 if the strings are identical; 0 otherwise. + */ +int ht_compare_string(const void *a, const void *b); + +#endif + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_2_8/exception.c b/libhdfs/hdfs_2_8/exception.c new file mode 100644 index 0000000..35e9d2d --- /dev/null +++ b/libhdfs/hdfs_2_8/exception.c @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "java.lang.UnsupportedOperationException", + 0, + ENOTSUP, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jstring jStr = NULL; + jvalue jVal; + jthrowable jthr; + const char *stackTrace; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "ExceptionUtils::getStackTrace error.)\n", className); + destroyLocalReference(env, jthr); + } else { + jStr = jVal.l; + stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL); + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "GetStringUTFChars error.)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + (*env)->ReleaseStringUTFChars(env, jStr, stackTrace); + } + } + } + destroyLocalReference(env, jStr); + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_2_8/exception.h b/libhdfs/hdfs_2_8/exception.h new file mode 100644 index 0000000..5fa7fa6 --- /dev/null +++ b/libhdfs/hdfs_2_8/exception.h @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * PrintExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Print out information about the pending exception and free it. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/libhdfs/hdfs_2_8/hdfs.c b/libhdfs/hdfs_2_8/hdfs.c new file mode 100644 index 0000000..7511f26 --- /dev/null +++ b/libhdfs/hdfs_2_8/hdfs.c @@ -0,0 +1,3342 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +int hdfsFileClearReadStatistics(hdfsFile file) +{ + jthrowable jthr; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return EINTERNAL; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "clearReadStatistics", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileClearReadStatistics: clearReadStatistics failed"); + goto done; + } + ret = 0; +done: + if (ret) { + errno = ret; + return ret; + } + return 0; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) +{ + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + jobject jPath = NULL; + + JNIEnv *env = getJNIEnv(); + + if (!env) { + errno = EINTERNAL; + return -1; + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): constructNewObjectOfPath", path); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "truncate", JMETHOD2(JPARAM(HADOOP_PATH), "J", "Z"), + jPath, newlength); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): FileSystem#truncate", path); + return -1; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +int hdfsUnbufferFile(hdfsFile file) +{ + int ret; + jthrowable jthr; + JNIEnv *env = getJNIEnv(); + + if (!env) { + ret = EINTERNAL; + goto done; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = ENOTSUP; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, HADOOP_ISTRM, + "unbuffer", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + HADOOP_ISTRM "#unbuffer failed:"); + goto done; + } + ret = 0; + +done: + errno = ret; + return ret; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + const char *interface; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + interface = (file->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + const char *interface; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + interface = (f->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + errno = 0; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + + + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_2.8.h b/libhdfs/hdfs_2_8/include/hdfs/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_2.8.h rename to libhdfs/hdfs_2_8/include/hdfs/hdfs.h diff --git a/libhdfs/hdfs_2_8/jni_helper.c b/libhdfs/hdfs_2_8/jni_helper.c new file mode 100644 index 0000000..50d9681 --- /dev/null +++ b/libhdfs/hdfs_2_8/jni_helper.c @@ -0,0 +1,595 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" +#include "platform.h" +#include "common/htable.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include + +static struct htable *gClassRefHTable = NULL; + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + jmethodID mid = 0; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jthrowable jthr = NULL; + jclass local_clazz = NULL; + jclass clazz = NULL; + int ret; + + mutexLock(&hdfsHashMutex); + if (!gClassRefHTable) { + gClassRefHTable = htable_alloc(MAX_HASH_TABLE_ELEM, ht_hash_string, + ht_compare_string); + if (!gClassRefHTable) { + jthr = newRuntimeError(env, "htable_alloc failed\n"); + goto done; + } + } + clazz = htable_get(gClassRefHTable, className); + if (clazz) { + *out = clazz; + goto done; + } + local_clazz = (*env)->FindClass(env,className); + if (!local_clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clazz = (*env)->NewGlobalRef(env, local_clazz); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + ret = htable_put(gClassRefHTable, (void*)className, clazz); + if (ret) { + jthr = newRuntimeError(env, "htable_put failed with error " + "code %d\n", ret); + goto done; + } + *out = clazz; + jthr = NULL; +done: + mutexUnlock(&hdfsHashMutex); + (*env)->DeleteLocalRef(env, local_clazz); + if (jthr && clazz) { + (*env)->DeleteGlobalRef(env, clazz); + } + return jthr; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + JNIEnv *env; + THREAD_LOCAL_STORAGE_GET_QUICK(); + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&env)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (env) { + mutexUnlock(&jvmMutex); + return env; + } + + env = getGlobalJNIEnv(); + mutexUnlock(&jvmMutex); + if (!env) { + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + return NULL; + } + if (threadLocalStorageSet(env)) { + return NULL; + } + THREAD_LOCAL_STORAGE_SET_QUICK(env); + return env; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_2_8/jni_helper.h b/libhdfs/hdfs_2_8/jni_helper.h new file mode 100644 index 0000000..90accc7 --- /dev/null +++ b/libhdfs/hdfs_2_8/jni_helper.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_2_8/os/mutexes.h b/libhdfs/hdfs_2_8/os/mutexes.h new file mode 100644 index 0000000..da30bf4 --- /dev/null +++ b/libhdfs/hdfs_2_8/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting the class reference hash table. */ +extern mutex hdfsHashMutex; + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/libhdfs/hdfs_2_8/os/posix/mutexes.c b/libhdfs/hdfs_2_8/os/posix/mutexes.c new file mode 100644 index 0000000..c4c2f26 --- /dev/null +++ b/libhdfs/hdfs_2_8/os/posix/mutexes.c @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +mutex jvmMutex = PTHREAD_MUTEX_INITIALIZER; + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/libhdfs/hdfs_2_8/os/posix/platform.h b/libhdfs/hdfs_2_8/os/posix/platform.h new file mode 100644 index 0000000..c63bbf9 --- /dev/null +++ b/libhdfs/hdfs_2_8/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/libhdfs/hdfs_2_8/os/posix/thread.c b/libhdfs/hdfs_2_8/os/posix/thread.c new file mode 100644 index 0000000..af0c61f --- /dev/null +++ b/libhdfs/hdfs_2_8/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/libhdfs/hdfs_2_8/os/posix/thread_local_storage.c b/libhdfs/hdfs_2_8/os/posix/thread_local_storage.c new file mode 100644 index 0000000..2f70e2c --- /dev/null +++ b/libhdfs/hdfs_2_8/os/posix/thread_local_storage.c @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +static void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + JNIEnv *env = v; + jint ret; + + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } +} + +int threadLocalStorageGet(JNIEnv **env) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *env = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(JNIEnv *env) +{ + int ret = pthread_setspecific(gTlsKey, env); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(env); + } + return ret; +} diff --git a/libhdfs/hdfs_2_8/os/thread.h b/libhdfs/hdfs_2_8/os/thread.h new file mode 100644 index 0000000..ae425d3 --- /dev/null +++ b/libhdfs/hdfs_2_8/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/libhdfs/hdfs_2_8/os/thread_local_storage.h b/libhdfs/hdfs_2_8/os/thread_local_storage.h new file mode 100644 index 0000000..a40d567 --- /dev/null +++ b/libhdfs/hdfs_2_8/os/thread_local_storage.h @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK() \ + static __thread JNIEnv *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + return quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(env) \ + { \ + quickTlsEnv = (env); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK() + #define THREAD_LOCAL_STORAGE_SET_QUICK(env) +#endif + +/** + * Gets the JNIEnv in thread-local storage for the current thread. If the call + * succeeds, and there is a JNIEnv associated with this thread, then returns 0 + * and populates env. If the call succeeds, but there is no JNIEnv associated + * with this thread, then returns 0 and sets JNIEnv to NULL. If the call fails, + * then returns non-zero. Only one thread at a time may execute this function. + * The caller is responsible for enforcing mutual exclusion. + * + * @param env JNIEnv out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(JNIEnv **env); + +/** + * Sets the JNIEnv in thread-local storage for the current thread. + * + * @param env JNIEnv to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(JNIEnv *env); + +#endif diff --git a/libhdfs/hdfs_2_8/os/windows/inttypes.h b/libhdfs/hdfs_2_8/os/windows/inttypes.h new file mode 100644 index 0000000..a520d15 --- /dev/null +++ b/libhdfs/hdfs_2_8/os/windows/inttypes.h @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_INTTYPES_H +#define LIBHDFS_INTTYPES_H + +/* On Windows, inttypes.h does not exist, so manually define what we need. */ + +#define PRId64 "I64d" +#define PRIu64 "I64u" +typedef unsigned __int64 uint64_t; + +#endif diff --git a/libhdfs/hdfs_2_8/os/windows/mutexes.c b/libhdfs/hdfs_2_8/os/windows/mutexes.c new file mode 100644 index 0000000..875f033 --- /dev/null +++ b/libhdfs/hdfs_2_8/os/windows/mutexes.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include + +mutex hdfsHashMutex; +mutex jvmMutex; + +/** + * Unfortunately, there is no simple static initializer for a critical section. + * Instead, the API requires calling InitializeCriticalSection. Since libhdfs + * lacks an explicit initialization function, there is no obvious existing place + * for the InitializeCriticalSection calls. To work around this, we define an + * initialization function and instruct the linker to set a pointer to that + * function as a user-defined global initializer. See discussion of CRT + * Initialization: + * http://msdn.microsoft.com/en-us/library/bb918180.aspx + */ +static void __cdecl initializeMutexes(void) { + InitializeCriticalSection(&hdfsHashMutex); + InitializeCriticalSection(&jvmMutex); +} +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) +const void (__cdecl *pInitialize)(void) = initializeMutexes; + +int mutexLock(mutex *m) { + EnterCriticalSection(m); + return 0; +} + +int mutexUnlock(mutex *m) { + LeaveCriticalSection(m); + return 0; +} diff --git a/libhdfs/hdfs_2_8/os/windows/platform.h b/libhdfs/hdfs_2_8/os/windows/platform.h new file mode 100644 index 0000000..9eedfde --- /dev/null +++ b/libhdfs/hdfs_2_8/os/windows/platform.h @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include +#include +#include + +/* + * O_ACCMODE defined to match Linux definition. + */ +#ifndef O_ACCMODE +#define O_ACCMODE 0x0003 +#endif + +/* + * Windows has a different name for its maximum path length constant. + */ +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +/* + * Windows does not define EDQUOT and ESTALE in errno.h. The closest equivalents + * are these constants from winsock.h. + */ +#ifndef EDQUOT +#define EDQUOT WSAEDQUOT +#endif + +#ifndef ESTALE +#define ESTALE WSAESTALE +#endif + +/* + * gcc-style type-checked format arguments are not supported on Windows, so just + * stub this macro. + */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) + +/* + * Define macros for various string formatting functions not defined on Windows. + * Where possible, we reroute to one of the secure CRT variants. On Windows, + * the preprocessor does support variadic macros, even though they weren't + * defined until C99. + */ +#define snprintf(str, size, format, ...) \ + _snprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) +#define strncpy(dest, src, n) \ + strncpy_s((dest), (n), (src), _TRUNCATE) +#define strtok_r(str, delim, saveptr) \ + strtok_s((str), (delim), (saveptr)) +#define vsnprintf(str, size, format, ...) \ + vsnprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) + +/* + * Mutex data type defined as Windows CRITICAL_SECTION. A critical section (not + * Windows mutex) is used, because libhdfs only needs synchronization of multiple + * threads within a single process, not synchronization across process + * boundaries. + */ +typedef CRITICAL_SECTION mutex; + +/* + * Thread data type defined as HANDLE to a Windows thread. + */ +typedef HANDLE threadId; + +#endif diff --git a/libhdfs/hdfs_2_8/os/windows/thread.c b/libhdfs/hdfs_2_8/os/windows/thread.c new file mode 100644 index 0000000..f5cc2a7 --- /dev/null +++ b/libhdfs/hdfs_2_8/os/windows/thread.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by CreateThread. + * + * @param toRun thread to run + * @return DWORD result of running thread (always 0) + */ +static DWORD WINAPI runThread(LPVOID toRun) { + const thread *t = toRun; + t->start(t->arg); + return 0; +} + +int threadCreate(thread *t) { + DWORD ret = 0; + HANDLE h; + h = CreateThread(NULL, 0, runThread, t, 0, NULL); + if (h) { + t->id = h; + } else { + ret = GetLastError(); + fprintf(stderr, "threadCreate: CreateThread failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + DWORD ret = WaitForSingleObject(t->id, INFINITE); + switch (ret) { + case WAIT_OBJECT_0: + break; + case WAIT_FAILED: + ret = GetLastError(); + fprintf(stderr, "threadJoin: WaitForSingleObject failed with error %d\n", + ret); + break; + default: + fprintf(stderr, "threadJoin: WaitForSingleObject unexpected error %d\n", + ret); + break; + } + return ret; +} diff --git a/libhdfs/hdfs_2_8/os/windows/thread_local_storage.c b/libhdfs/hdfs_2_8/os/windows/thread_local_storage.c new file mode 100644 index 0000000..4c415e1 --- /dev/null +++ b/libhdfs/hdfs_2_8/os/windows/thread_local_storage.c @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static DWORD gTlsIndex = TLS_OUT_OF_INDEXES; + +/** + * If the current thread has a JNIEnv in thread-local storage, then detaches the + * current thread from the JVM. + */ +static void detachCurrentThreadFromJvm() +{ + JNIEnv *env = NULL; + JavaVM *vm; + jint ret; + if (threadLocalStorageGet(&env) || !env) { + return; + } + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, + "detachCurrentThreadFromJvm: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } +} + +/** + * Unlike pthreads, the Windows API does not seem to provide a convenient way to + * hook a callback onto thread shutdown. However, the Windows portable + * executable format does define a concept of thread-local storage callbacks. + * Here, we define a function and instruct the linker to set a pointer to that + * function in the segment for thread-local storage callbacks. See page 85 of + * Microsoft Portable Executable and Common Object File Format Specification: + * http://msdn.microsoft.com/en-us/gg463119.aspx + * This technique only works for implicit linking (OS loads DLL on demand), not + * for explicit linking (user code calls LoadLibrary directly). This effectively + * means that we have a known limitation: libhdfs may not work correctly if a + * Windows application attempts to use it via explicit linking. + * + * @param h module handle + * @param reason the reason for calling the callback + * @param pv reserved, unused + */ +static void NTAPI tlsCallback(PVOID h, DWORD reason, PVOID pv) +{ + DWORD tlsIndex; + switch (reason) { + case DLL_THREAD_DETACH: + detachCurrentThreadFromJvm(); + break; + case DLL_PROCESS_DETACH: + detachCurrentThreadFromJvm(); + tlsIndex = gTlsIndex; + gTlsIndex = TLS_OUT_OF_INDEXES; + if (!TlsFree(tlsIndex)) { + fprintf(stderr, "tlsCallback: TlsFree failed with error %d\n", + GetLastError()); + } + break; + default: + break; + } +} + +/* + * A variable named _tls_used contains the TLS directory, which contains a list + * of pointers to callback functions. Normally, the linker won't retain this + * variable unless the executable has implicit thread-local variables, defined + * using the __declspec(thread) extended storage-class modifier. libhdfs + * doesn't use __declspec(thread), and we have no guarantee that the executable + * linked to libhdfs will use __declspec(thread). By forcing the linker to + * reference _tls_used, we guarantee that the binary retains the TLS directory. + * See Microsoft Visual Studio 10.0/VC/crt/src/tlssup.c . + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#endif + +/* + * We must retain a pointer to the callback function. Force the linker to keep + * this symbol, even though it appears that nothing in our source code uses it. + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:pTlsCallback") +#else +#pragma comment(linker, "/INCLUDE:_pTlsCallback") +#endif + +/* + * Define constant pointer to our callback, and tell the linker to pin it into + * the TLS directory so that it receives thread callbacks. Use external linkage + * to protect against the linker discarding the seemingly unused symbol. + */ +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK pTlsCallback; +const PIMAGE_TLS_CALLBACK pTlsCallback = tlsCallback; +#pragma const_seg() + +int threadLocalStorageGet(JNIEnv **env) +{ + LPVOID tls; + DWORD ret; + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + gTlsIndex = TlsAlloc(); + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + fprintf(stderr, + "threadLocalStorageGet: TlsAlloc failed with error %d\n", + TLS_OUT_OF_INDEXES); + return TLS_OUT_OF_INDEXES; + } + } + tls = TlsGetValue(gTlsIndex); + if (tls) { + *env = tls; + return 0; + } else { + ret = GetLastError(); + if (ERROR_SUCCESS == ret) { + /* Thread-local storage contains NULL, because we haven't set it yet. */ + *env = NULL; + return 0; + } else { + /* + * The API call failed. According to documentation, TlsGetValue cannot + * fail as long as the index is a valid index from a successful TlsAlloc + * call. This error handling is purely defensive. + */ + fprintf(stderr, + "threadLocalStorageGet: TlsGetValue failed with error %d\n", ret); + return ret; + } + } +} + +int threadLocalStorageSet(JNIEnv *env) +{ + DWORD ret = 0; + if (!TlsSetValue(gTlsIndex, (LPVOID)env)) { + ret = GetLastError(); + fprintf(stderr, + "threadLocalStorageSet: TlsSetValue failed with error %d\n", + ret); + detachCurrentThreadFromJvm(env); + } + return ret; +} diff --git a/libhdfs/hdfs_2_8/os/windows/unistd.h b/libhdfs/hdfs_2_8/os/windows/unistd.h new file mode 100644 index 0000000..b82ce48 --- /dev/null +++ b/libhdfs/hdfs_2_8/os/windows/unistd.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_UNISTD_H +#define LIBHDFS_UNISTD_H + +/* On Windows, unistd.h does not exist, so manually define what we need. */ + +#include /* Declares getpid(). */ +#include + +/* Re-route sleep to Sleep, converting units from seconds to milliseconds. */ +#define sleep(seconds) Sleep((seconds) * 1000) +#endif diff --git a/libhdfs/hdfs_2_9/common/htable.c b/libhdfs/hdfs_2_9/common/htable.c new file mode 100644 index 0000000..50c89ea --- /dev/null +++ b/libhdfs/hdfs_2_9/common/htable.c @@ -0,0 +1,287 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/htable.h" + +#include +#include +#include +#include +#include + +struct htable_pair { + void *key; + void *val; +}; + +/** + * A hash table which uses linear probing. + */ +struct htable { + uint32_t capacity; + uint32_t used; + htable_hash_fn_t hash_fun; + htable_eq_fn_t eq_fun; + struct htable_pair *elem; +}; + +/** + * An internal function for inserting a value into the hash table. + * + * Note: this function assumes that you have made enough space in the table. + * + * @param nelem The new element to insert. + * @param capacity The capacity of the hash table. + * @param hash_fun The hash function to use. + * @param key The key to insert. + * @param val The value to insert. + */ +static void htable_insert_internal(struct htable_pair *nelem, + uint32_t capacity, htable_hash_fn_t hash_fun, void *key, + void *val) +{ + uint32_t i; + + i = hash_fun(key, capacity); + while (1) { + if (!nelem[i].key) { + nelem[i].key = key; + nelem[i].val = val; + return; + } + i++; + if (i == capacity) { + i = 0; + } + } +} + +static int htable_realloc(struct htable *htable, uint32_t new_capacity) +{ + struct htable_pair *nelem; + uint32_t i, old_capacity = htable->capacity; + htable_hash_fn_t hash_fun = htable->hash_fun; + + nelem = calloc(new_capacity, sizeof(struct htable_pair)); + if (!nelem) { + return ENOMEM; + } + for (i = 0; i < old_capacity; i++) { + struct htable_pair *pair = htable->elem + i; + if (pair->key) { + htable_insert_internal(nelem, new_capacity, hash_fun, + pair->key, pair->val); + } + } + free(htable->elem); + htable->elem = nelem; + htable->capacity = new_capacity; + return 0; +} + +static uint32_t round_up_to_power_of_2(uint32_t i) +{ + if (i == 0) { + return 1; + } + i--; + i |= i >> 1; + i |= i >> 2; + i |= i >> 4; + i |= i >> 8; + i |= i >> 16; + i++; + return i; +} + +struct htable *htable_alloc(uint32_t size, + htable_hash_fn_t hash_fun, htable_eq_fn_t eq_fun) +{ + struct htable *htable; + + htable = calloc(1, sizeof(*htable)); + if (!htable) { + return NULL; + } + size = round_up_to_power_of_2(size); + if (size < HTABLE_MIN_SIZE) { + size = HTABLE_MIN_SIZE; + } + htable->hash_fun = hash_fun; + htable->eq_fun = eq_fun; + htable->used = 0; + if (htable_realloc(htable, size)) { + free(htable); + return NULL; + } + return htable; +} + +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx) +{ + uint32_t i; + + for (i = 0; i != htable->capacity; ++i) { + struct htable_pair *elem = htable->elem + i; + if (elem->key) { + fun(ctx, elem->key, elem->val); + } + } +} + +void htable_free(struct htable *htable) +{ + if (htable) { + free(htable->elem); + free(htable); + } +} + +int htable_put(struct htable *htable, void *key, void *val) +{ + int ret; + uint32_t nused; + + // NULL is not a valid key value. + // This helps us implement htable_get_internal efficiently, since we know + // that we can stop when we encounter the first NULL key. + if (!key) { + return EINVAL; + } + // NULL is not a valid value. Otherwise the results of htable_get would + // be confusing (does a NULL return mean entry not found, or that the + // entry was found and was NULL?) + if (!val) { + return EINVAL; + } + // Re-hash if we have used more than half of the hash table + nused = htable->used + 1; + if (nused >= (htable->capacity / 2)) { + ret = htable_realloc(htable, htable->capacity * 2); + if (ret) + return ret; + } + htable_insert_internal(htable->elem, htable->capacity, + htable->hash_fun, key, val); + htable->used++; + return 0; +} + +static int htable_get_internal(const struct htable *htable, + const void *key, uint32_t *out) +{ + uint32_t start_idx, idx; + + start_idx = htable->hash_fun(key, htable->capacity); + idx = start_idx; + while (1) { + struct htable_pair *pair = htable->elem + idx; + if (!pair->key) { + // We always maintain the invariant that the entries corresponding + // to a given key are stored in a contiguous block, not separated + // by any NULLs. So if we encounter a NULL, our search is over. + return ENOENT; + } else if (htable->eq_fun(pair->key, key)) { + *out = idx; + return 0; + } + idx++; + if (idx == htable->capacity) { + idx = 0; + } + if (idx == start_idx) { + return ENOENT; + } + } +} + +void *htable_get(const struct htable *htable, const void *key) +{ + uint32_t idx; + + if (htable_get_internal(htable, key, &idx)) { + return NULL; + } + return htable->elem[idx].val; +} + +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val) +{ + uint32_t hole, i; + const void *nkey; + + if (htable_get_internal(htable, key, &hole)) { + *found_key = NULL; + *found_val = NULL; + return; + } + i = hole; + htable->used--; + // We need to maintain the compactness invariant used in + // htable_get_internal. This invariant specifies that the entries for any + // given key are never separated by NULLs (although they may be separated + // by entries for other keys.) + while (1) { + i++; + if (i == htable->capacity) { + i = 0; + } + nkey = htable->elem[i].key; + if (!nkey) { + *found_key = htable->elem[hole].key; + *found_val = htable->elem[hole].val; + htable->elem[hole].key = NULL; + htable->elem[hole].val = NULL; + return; + } else if (htable->eq_fun(key, nkey)) { + htable->elem[hole].key = htable->elem[i].key; + htable->elem[hole].val = htable->elem[i].val; + hole = i; + } + } +} + +uint32_t htable_used(const struct htable *htable) +{ + return htable->used; +} + +uint32_t htable_capacity(const struct htable *htable) +{ + return htable->capacity; +} + +uint32_t ht_hash_string(const void *str, uint32_t max) +{ + const char *s = str; + uint32_t hash = 0; + + while (*s) { + hash = (hash * 31) + *s; + s++; + } + return hash % max; +} + +int ht_compare_string(const void *a, const void *b) +{ + return strcmp(a, b) == 0; +} + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_2_9/common/htable.h b/libhdfs/hdfs_2_9/common/htable.h new file mode 100644 index 0000000..33f1229 --- /dev/null +++ b/libhdfs/hdfs_2_9/common/htable.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HADOOP_CORE_COMMON_HASH_TABLE +#define HADOOP_CORE_COMMON_HASH_TABLE + +#include +#include +#include + +#define HTABLE_MIN_SIZE 4 + +struct htable; + +/** + * An HTable hash function. + * + * @param key The key. + * @param capacity The total capacity. + * + * @return The hash slot. Must be less than the capacity. + */ +typedef uint32_t (*htable_hash_fn_t)(const void *key, uint32_t capacity); + +/** + * An HTable equality function. Compares two keys. + * + * @param a First key. + * @param b Second key. + * + * @return nonzero if the keys are equal. + */ +typedef int (*htable_eq_fn_t)(const void *a, const void *b); + +/** + * Allocate a new hash table. + * + * @param capacity The minimum suggested starting capacity. + * @param hash_fun The hash function to use in this hash table. + * @param eq_fun The equals function to use in this hash table. + * + * @return The new hash table on success; NULL on OOM. + */ +struct htable *htable_alloc(uint32_t capacity, htable_hash_fn_t hash_fun, + htable_eq_fn_t eq_fun); + +typedef void (*visitor_fn_t)(void *ctx, void *key, void *val); + +/** + * Visit all of the entries in the hash table. + * + * @param htable The hash table. + * @param fun The callback function to invoke on each key and value. + * @param ctx Context pointer to pass to the callback. + */ +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx); + +/** + * Free the hash table. + * + * It is up the calling code to ensure that the keys and values inside the + * table are de-allocated, if that is necessary. + * + * @param htable The hash table. + */ +void htable_free(struct htable *htable); + +/** + * Add an entry to the hash table. + * + * @param htable The hash table. + * @param key The key to add. This cannot be NULL. + * @param fun The value to add. This cannot be NULL. + * + * @return 0 on success; + * EEXIST if the value already exists in the table; + * ENOMEM if there is not enough memory to add the element. + * EFBIG if the hash table has too many entries to fit in 32 + * bits. + */ +int htable_put(struct htable *htable, void *key, void *val); + +/** + * Get an entry from the hash table. + * + * @param htable The hash table. + * @param key The key to find. + * + * @return NULL if there is no such entry; the entry otherwise. + */ +void *htable_get(const struct htable *htable, const void *key); + +/** + * Get an entry from the hash table and remove it. + * + * @param htable The hash table. + * @param key The key for the entry find and remove. + * @param found_key (out param) NULL if the entry was not found; the found key + * otherwise. + * @param found_val (out param) NULL if the entry was not found; the found + * value otherwise. + */ +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val); + +/** + * Get the number of entries used in the hash table. + * + * @param htable The hash table. + * + * @return The number of entries used in the hash table. + */ +uint32_t htable_used(const struct htable *htable); + +/** + * Get the capacity of the hash table. + * + * @param htable The hash table. + * + * @return The capacity of the hash table. + */ +uint32_t htable_capacity(const struct htable *htable); + +/** + * Hash a string. + * + * @param str The string. + * @param max Maximum hash value + * + * @return A number less than max. + */ +uint32_t ht_hash_string(const void *str, uint32_t max); + +/** + * Compare two strings. + * + * @param a The first string. + * @param b The second string. + * + * @return 1 if the strings are identical; 0 otherwise. + */ +int ht_compare_string(const void *a, const void *b); + +#endif + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_2_9/exception.c b/libhdfs/hdfs_2_9/exception.c new file mode 100644 index 0000000..35e9d2d --- /dev/null +++ b/libhdfs/hdfs_2_9/exception.c @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "java.lang.UnsupportedOperationException", + 0, + ENOTSUP, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jstring jStr = NULL; + jvalue jVal; + jthrowable jthr; + const char *stackTrace; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + "getStackTrace", "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "ExceptionUtils::getStackTrace error.)\n", className); + destroyLocalReference(env, jthr); + } else { + jStr = jVal.l; + stackTrace = (*env)->GetStringUTFChars(env, jStr, NULL); + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s exception: " + "GetStringUTFChars error.)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + (*env)->ReleaseStringUTFChars(env, jStr, stackTrace); + } + } + } + destroyLocalReference(env, jStr); + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_2_9/exception.h b/libhdfs/hdfs_2_9/exception.h new file mode 100644 index 0000000..5fa7fa6 --- /dev/null +++ b/libhdfs/hdfs_2_9/exception.h @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * PrintExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Print out information about an exception and free it. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Print out information about the pending exception and free it. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/libhdfs/hdfs_2_9/hdfs.c b/libhdfs/hdfs_2_9/hdfs.c new file mode 100644 index 0000000..ac4bca6 --- /dev/null +++ b/libhdfs/hdfs_2_9/hdfs.c @@ -0,0 +1,3438 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/DFSInputStream$ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/DFSInputStream$ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +int hdfsFileClearReadStatistics(hdfsFile file) +{ + jthrowable jthr; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return EINTERNAL; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "clearReadStatistics", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileClearReadStatistics: clearReadStatistics failed"); + goto done; + } + ret = 0; +done: + if (ret) { + errno = ret; + return ret; + } + return 0; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + struct hdfsStreamBuilder *bld = hdfsStreamBuilderAlloc(fs, path, flags); + if (bufferSize != 0) { + hdfsStreamBuilderSetBufferSize(bld, bufferSize); + } + if (replication != 0) { + hdfsStreamBuilderSetReplication(bld, replication); + } + if (blockSize != 0) { + hdfsStreamBuilderSetDefaultBlockSize(bld, blockSize); + } + return hdfsStreamBuilderBuild(bld); +} + +struct hdfsStreamBuilder { + hdfsFS fs; + int flags; + int32_t bufferSize; + int16_t replication; + int64_t defaultBlockSize; + char path[1]; +}; + +struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs, + const char *path, int flags) +{ + int path_len = strlen(path); + struct hdfsStreamBuilder *bld; + + // sizeof(hdfsStreamBuilder->path) includes one byte for the string + // terminator + bld = malloc(sizeof(struct hdfsStreamBuilder) + path_len); + if (!bld) { + errno = ENOMEM; + return NULL; + } + bld->fs = fs; + bld->flags = flags; + bld->bufferSize = 0; + bld->replication = 0; + bld->defaultBlockSize = 0; + memcpy(bld->path, path, path_len); + bld->path[path_len] = '\0'; + return bld; +} + +void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld) +{ + free(bld); +} + +int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld, + int32_t bufferSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->bufferSize = bufferSize; + return 0; +} + +int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld, + int16_t replication) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->replication = replication; + return 0; +} + +int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld, + int64_t defaultBlockSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->defaultBlockSize = defaultBlockSize; + return 0; +} + +static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, + int32_t bufferSize, int16_t replication, int64_t blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld) +{ + hdfsFile file = hdfsOpenFileImpl(bld->fs, bld->path, bld->flags, + bld->bufferSize, bld->replication, bld->defaultBlockSize); + int prevErrno = errno; + hdfsStreamBuilderFree(bld); + errno = prevErrno; + return file; +} + +int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) +{ + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + jobject jPath = NULL; + + JNIEnv *env = getJNIEnv(); + + if (!env) { + errno = EINTERNAL; + return -1; + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): constructNewObjectOfPath", path); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "truncate", JMETHOD2(JPARAM(HADOOP_PATH), "J", "Z"), + jPath, newlength); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): FileSystem#truncate", path); + return -1; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +int hdfsUnbufferFile(hdfsFile file) +{ + int ret; + jthrowable jthr; + JNIEnv *env = getJNIEnv(); + + if (!env) { + ret = EINTERNAL; + goto done; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = ENOTSUP; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, HADOOP_ISTRM, + "unbuffer", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + HADOOP_ISTRM "#unbuffer failed:"); + goto done; + } + ret = 0; + +done: + errno = ret; + return ret; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + const char *interface; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + interface = (file->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + const char *interface; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + interface = (f->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + errno = 0; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + + + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_2.9.h b/libhdfs/hdfs_2_9/include/hdfs/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_2.9.h rename to libhdfs/hdfs_2_9/include/hdfs/hdfs.h diff --git a/libhdfs/hdfs_2_9/jni_helper.c b/libhdfs/hdfs_2_9/jni_helper.c new file mode 100644 index 0000000..50d9681 --- /dev/null +++ b/libhdfs/hdfs_2_9/jni_helper.c @@ -0,0 +1,595 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" +#include "platform.h" +#include "common/htable.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include + +static struct htable *gClassRefHTable = NULL; + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + jmethodID mid = 0; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jthrowable jthr = NULL; + jclass local_clazz = NULL; + jclass clazz = NULL; + int ret; + + mutexLock(&hdfsHashMutex); + if (!gClassRefHTable) { + gClassRefHTable = htable_alloc(MAX_HASH_TABLE_ELEM, ht_hash_string, + ht_compare_string); + if (!gClassRefHTable) { + jthr = newRuntimeError(env, "htable_alloc failed\n"); + goto done; + } + } + clazz = htable_get(gClassRefHTable, className); + if (clazz) { + *out = clazz; + goto done; + } + local_clazz = (*env)->FindClass(env,className); + if (!local_clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clazz = (*env)->NewGlobalRef(env, local_clazz); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + ret = htable_put(gClassRefHTable, (void*)className, clazz); + if (ret) { + jthr = newRuntimeError(env, "htable_put failed with error " + "code %d\n", ret); + goto done; + } + *out = clazz; + jthr = NULL; +done: + mutexUnlock(&hdfsHashMutex); + (*env)->DeleteLocalRef(env, local_clazz); + if (jthr && clazz) { + (*env)->DeleteGlobalRef(env, clazz); + } + return jthr; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + JNIEnv *env; + THREAD_LOCAL_STORAGE_GET_QUICK(); + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&env)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (env) { + mutexUnlock(&jvmMutex); + return env; + } + + env = getGlobalJNIEnv(); + mutexUnlock(&jvmMutex); + if (!env) { + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + return NULL; + } + if (threadLocalStorageSet(env)) { + return NULL; + } + THREAD_LOCAL_STORAGE_SET_QUICK(env); + return env; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_2_9/jni_helper.h b/libhdfs/hdfs_2_9/jni_helper.h new file mode 100644 index 0000000..90accc7 --- /dev/null +++ b/libhdfs/hdfs_2_9/jni_helper.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_2_9/os/mutexes.h b/libhdfs/hdfs_2_9/os/mutexes.h new file mode 100644 index 0000000..da30bf4 --- /dev/null +++ b/libhdfs/hdfs_2_9/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting the class reference hash table. */ +extern mutex hdfsHashMutex; + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/libhdfs/hdfs_2_9/os/posix/mutexes.c b/libhdfs/hdfs_2_9/os/posix/mutexes.c new file mode 100644 index 0000000..c4c2f26 --- /dev/null +++ b/libhdfs/hdfs_2_9/os/posix/mutexes.c @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +mutex jvmMutex = PTHREAD_MUTEX_INITIALIZER; + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/libhdfs/hdfs_2_9/os/posix/platform.h b/libhdfs/hdfs_2_9/os/posix/platform.h new file mode 100644 index 0000000..c63bbf9 --- /dev/null +++ b/libhdfs/hdfs_2_9/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/libhdfs/hdfs_2_9/os/posix/thread.c b/libhdfs/hdfs_2_9/os/posix/thread.c new file mode 100644 index 0000000..af0c61f --- /dev/null +++ b/libhdfs/hdfs_2_9/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/libhdfs/hdfs_2_9/os/posix/thread_local_storage.c b/libhdfs/hdfs_2_9/os/posix/thread_local_storage.c new file mode 100644 index 0000000..2f70e2c --- /dev/null +++ b/libhdfs/hdfs_2_9/os/posix/thread_local_storage.c @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +static void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + JNIEnv *env = v; + jint ret; + + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } +} + +int threadLocalStorageGet(JNIEnv **env) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *env = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(JNIEnv *env) +{ + int ret = pthread_setspecific(gTlsKey, env); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(env); + } + return ret; +} diff --git a/libhdfs/hdfs_2_9/os/thread.h b/libhdfs/hdfs_2_9/os/thread.h new file mode 100644 index 0000000..ae425d3 --- /dev/null +++ b/libhdfs/hdfs_2_9/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/libhdfs/hdfs_2_9/os/thread_local_storage.h b/libhdfs/hdfs_2_9/os/thread_local_storage.h new file mode 100644 index 0000000..a40d567 --- /dev/null +++ b/libhdfs/hdfs_2_9/os/thread_local_storage.h @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK() \ + static __thread JNIEnv *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + return quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(env) \ + { \ + quickTlsEnv = (env); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK() + #define THREAD_LOCAL_STORAGE_SET_QUICK(env) +#endif + +/** + * Gets the JNIEnv in thread-local storage for the current thread. If the call + * succeeds, and there is a JNIEnv associated with this thread, then returns 0 + * and populates env. If the call succeeds, but there is no JNIEnv associated + * with this thread, then returns 0 and sets JNIEnv to NULL. If the call fails, + * then returns non-zero. Only one thread at a time may execute this function. + * The caller is responsible for enforcing mutual exclusion. + * + * @param env JNIEnv out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(JNIEnv **env); + +/** + * Sets the JNIEnv in thread-local storage for the current thread. + * + * @param env JNIEnv to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(JNIEnv *env); + +#endif diff --git a/libhdfs/hdfs_2_9/os/windows/inttypes.h b/libhdfs/hdfs_2_9/os/windows/inttypes.h new file mode 100644 index 0000000..a520d15 --- /dev/null +++ b/libhdfs/hdfs_2_9/os/windows/inttypes.h @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_INTTYPES_H +#define LIBHDFS_INTTYPES_H + +/* On Windows, inttypes.h does not exist, so manually define what we need. */ + +#define PRId64 "I64d" +#define PRIu64 "I64u" +typedef unsigned __int64 uint64_t; + +#endif diff --git a/libhdfs/hdfs_2_9/os/windows/mutexes.c b/libhdfs/hdfs_2_9/os/windows/mutexes.c new file mode 100644 index 0000000..875f033 --- /dev/null +++ b/libhdfs/hdfs_2_9/os/windows/mutexes.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include + +mutex hdfsHashMutex; +mutex jvmMutex; + +/** + * Unfortunately, there is no simple static initializer for a critical section. + * Instead, the API requires calling InitializeCriticalSection. Since libhdfs + * lacks an explicit initialization function, there is no obvious existing place + * for the InitializeCriticalSection calls. To work around this, we define an + * initialization function and instruct the linker to set a pointer to that + * function as a user-defined global initializer. See discussion of CRT + * Initialization: + * http://msdn.microsoft.com/en-us/library/bb918180.aspx + */ +static void __cdecl initializeMutexes(void) { + InitializeCriticalSection(&hdfsHashMutex); + InitializeCriticalSection(&jvmMutex); +} +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) +const void (__cdecl *pInitialize)(void) = initializeMutexes; + +int mutexLock(mutex *m) { + EnterCriticalSection(m); + return 0; +} + +int mutexUnlock(mutex *m) { + LeaveCriticalSection(m); + return 0; +} diff --git a/libhdfs/hdfs_2_9/os/windows/platform.h b/libhdfs/hdfs_2_9/os/windows/platform.h new file mode 100644 index 0000000..9eedfde --- /dev/null +++ b/libhdfs/hdfs_2_9/os/windows/platform.h @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include +#include +#include + +/* + * O_ACCMODE defined to match Linux definition. + */ +#ifndef O_ACCMODE +#define O_ACCMODE 0x0003 +#endif + +/* + * Windows has a different name for its maximum path length constant. + */ +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +/* + * Windows does not define EDQUOT and ESTALE in errno.h. The closest equivalents + * are these constants from winsock.h. + */ +#ifndef EDQUOT +#define EDQUOT WSAEDQUOT +#endif + +#ifndef ESTALE +#define ESTALE WSAESTALE +#endif + +/* + * gcc-style type-checked format arguments are not supported on Windows, so just + * stub this macro. + */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) + +/* + * Define macros for various string formatting functions not defined on Windows. + * Where possible, we reroute to one of the secure CRT variants. On Windows, + * the preprocessor does support variadic macros, even though they weren't + * defined until C99. + */ +#define snprintf(str, size, format, ...) \ + _snprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) +#define strncpy(dest, src, n) \ + strncpy_s((dest), (n), (src), _TRUNCATE) +#define strtok_r(str, delim, saveptr) \ + strtok_s((str), (delim), (saveptr)) +#define vsnprintf(str, size, format, ...) \ + vsnprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) + +/* + * Mutex data type defined as Windows CRITICAL_SECTION. A critical section (not + * Windows mutex) is used, because libhdfs only needs synchronization of multiple + * threads within a single process, not synchronization across process + * boundaries. + */ +typedef CRITICAL_SECTION mutex; + +/* + * Thread data type defined as HANDLE to a Windows thread. + */ +typedef HANDLE threadId; + +#endif diff --git a/libhdfs/hdfs_2_9/os/windows/thread.c b/libhdfs/hdfs_2_9/os/windows/thread.c new file mode 100644 index 0000000..f5cc2a7 --- /dev/null +++ b/libhdfs/hdfs_2_9/os/windows/thread.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by CreateThread. + * + * @param toRun thread to run + * @return DWORD result of running thread (always 0) + */ +static DWORD WINAPI runThread(LPVOID toRun) { + const thread *t = toRun; + t->start(t->arg); + return 0; +} + +int threadCreate(thread *t) { + DWORD ret = 0; + HANDLE h; + h = CreateThread(NULL, 0, runThread, t, 0, NULL); + if (h) { + t->id = h; + } else { + ret = GetLastError(); + fprintf(stderr, "threadCreate: CreateThread failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + DWORD ret = WaitForSingleObject(t->id, INFINITE); + switch (ret) { + case WAIT_OBJECT_0: + break; + case WAIT_FAILED: + ret = GetLastError(); + fprintf(stderr, "threadJoin: WaitForSingleObject failed with error %d\n", + ret); + break; + default: + fprintf(stderr, "threadJoin: WaitForSingleObject unexpected error %d\n", + ret); + break; + } + return ret; +} diff --git a/libhdfs/hdfs_2_9/os/windows/thread_local_storage.c b/libhdfs/hdfs_2_9/os/windows/thread_local_storage.c new file mode 100644 index 0000000..4c415e1 --- /dev/null +++ b/libhdfs/hdfs_2_9/os/windows/thread_local_storage.c @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static DWORD gTlsIndex = TLS_OUT_OF_INDEXES; + +/** + * If the current thread has a JNIEnv in thread-local storage, then detaches the + * current thread from the JVM. + */ +static void detachCurrentThreadFromJvm() +{ + JNIEnv *env = NULL; + JavaVM *vm; + jint ret; + if (threadLocalStorageGet(&env) || !env) { + return; + } + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, + "detachCurrentThreadFromJvm: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } +} + +/** + * Unlike pthreads, the Windows API does not seem to provide a convenient way to + * hook a callback onto thread shutdown. However, the Windows portable + * executable format does define a concept of thread-local storage callbacks. + * Here, we define a function and instruct the linker to set a pointer to that + * function in the segment for thread-local storage callbacks. See page 85 of + * Microsoft Portable Executable and Common Object File Format Specification: + * http://msdn.microsoft.com/en-us/gg463119.aspx + * This technique only works for implicit linking (OS loads DLL on demand), not + * for explicit linking (user code calls LoadLibrary directly). This effectively + * means that we have a known limitation: libhdfs may not work correctly if a + * Windows application attempts to use it via explicit linking. + * + * @param h module handle + * @param reason the reason for calling the callback + * @param pv reserved, unused + */ +static void NTAPI tlsCallback(PVOID h, DWORD reason, PVOID pv) +{ + DWORD tlsIndex; + switch (reason) { + case DLL_THREAD_DETACH: + detachCurrentThreadFromJvm(); + break; + case DLL_PROCESS_DETACH: + detachCurrentThreadFromJvm(); + tlsIndex = gTlsIndex; + gTlsIndex = TLS_OUT_OF_INDEXES; + if (!TlsFree(tlsIndex)) { + fprintf(stderr, "tlsCallback: TlsFree failed with error %d\n", + GetLastError()); + } + break; + default: + break; + } +} + +/* + * A variable named _tls_used contains the TLS directory, which contains a list + * of pointers to callback functions. Normally, the linker won't retain this + * variable unless the executable has implicit thread-local variables, defined + * using the __declspec(thread) extended storage-class modifier. libhdfs + * doesn't use __declspec(thread), and we have no guarantee that the executable + * linked to libhdfs will use __declspec(thread). By forcing the linker to + * reference _tls_used, we guarantee that the binary retains the TLS directory. + * See Microsoft Visual Studio 10.0/VC/crt/src/tlssup.c . + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#endif + +/* + * We must retain a pointer to the callback function. Force the linker to keep + * this symbol, even though it appears that nothing in our source code uses it. + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:pTlsCallback") +#else +#pragma comment(linker, "/INCLUDE:_pTlsCallback") +#endif + +/* + * Define constant pointer to our callback, and tell the linker to pin it into + * the TLS directory so that it receives thread callbacks. Use external linkage + * to protect against the linker discarding the seemingly unused symbol. + */ +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK pTlsCallback; +const PIMAGE_TLS_CALLBACK pTlsCallback = tlsCallback; +#pragma const_seg() + +int threadLocalStorageGet(JNIEnv **env) +{ + LPVOID tls; + DWORD ret; + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + gTlsIndex = TlsAlloc(); + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + fprintf(stderr, + "threadLocalStorageGet: TlsAlloc failed with error %d\n", + TLS_OUT_OF_INDEXES); + return TLS_OUT_OF_INDEXES; + } + } + tls = TlsGetValue(gTlsIndex); + if (tls) { + *env = tls; + return 0; + } else { + ret = GetLastError(); + if (ERROR_SUCCESS == ret) { + /* Thread-local storage contains NULL, because we haven't set it yet. */ + *env = NULL; + return 0; + } else { + /* + * The API call failed. According to documentation, TlsGetValue cannot + * fail as long as the index is a valid index from a successful TlsAlloc + * call. This error handling is purely defensive. + */ + fprintf(stderr, + "threadLocalStorageGet: TlsGetValue failed with error %d\n", ret); + return ret; + } + } +} + +int threadLocalStorageSet(JNIEnv *env) +{ + DWORD ret = 0; + if (!TlsSetValue(gTlsIndex, (LPVOID)env)) { + ret = GetLastError(); + fprintf(stderr, + "threadLocalStorageSet: TlsSetValue failed with error %d\n", + ret); + detachCurrentThreadFromJvm(env); + } + return ret; +} diff --git a/libhdfs/hdfs_2_9/os/windows/unistd.h b/libhdfs/hdfs_2_9/os/windows/unistd.h new file mode 100644 index 0000000..b82ce48 --- /dev/null +++ b/libhdfs/hdfs_2_9/os/windows/unistd.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_UNISTD_H +#define LIBHDFS_UNISTD_H + +/* On Windows, unistd.h does not exist, so manually define what we need. */ + +#include /* Declares getpid(). */ +#include + +/* Re-route sleep to Sleep, converting units from seconds to milliseconds. */ +#define sleep(seconds) Sleep((seconds) * 1000) +#endif diff --git a/libhdfs/hdfs_3_0/common/htable.c b/libhdfs/hdfs_3_0/common/htable.c new file mode 100644 index 0000000..50c89ea --- /dev/null +++ b/libhdfs/hdfs_3_0/common/htable.c @@ -0,0 +1,287 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/htable.h" + +#include +#include +#include +#include +#include + +struct htable_pair { + void *key; + void *val; +}; + +/** + * A hash table which uses linear probing. + */ +struct htable { + uint32_t capacity; + uint32_t used; + htable_hash_fn_t hash_fun; + htable_eq_fn_t eq_fun; + struct htable_pair *elem; +}; + +/** + * An internal function for inserting a value into the hash table. + * + * Note: this function assumes that you have made enough space in the table. + * + * @param nelem The new element to insert. + * @param capacity The capacity of the hash table. + * @param hash_fun The hash function to use. + * @param key The key to insert. + * @param val The value to insert. + */ +static void htable_insert_internal(struct htable_pair *nelem, + uint32_t capacity, htable_hash_fn_t hash_fun, void *key, + void *val) +{ + uint32_t i; + + i = hash_fun(key, capacity); + while (1) { + if (!nelem[i].key) { + nelem[i].key = key; + nelem[i].val = val; + return; + } + i++; + if (i == capacity) { + i = 0; + } + } +} + +static int htable_realloc(struct htable *htable, uint32_t new_capacity) +{ + struct htable_pair *nelem; + uint32_t i, old_capacity = htable->capacity; + htable_hash_fn_t hash_fun = htable->hash_fun; + + nelem = calloc(new_capacity, sizeof(struct htable_pair)); + if (!nelem) { + return ENOMEM; + } + for (i = 0; i < old_capacity; i++) { + struct htable_pair *pair = htable->elem + i; + if (pair->key) { + htable_insert_internal(nelem, new_capacity, hash_fun, + pair->key, pair->val); + } + } + free(htable->elem); + htable->elem = nelem; + htable->capacity = new_capacity; + return 0; +} + +static uint32_t round_up_to_power_of_2(uint32_t i) +{ + if (i == 0) { + return 1; + } + i--; + i |= i >> 1; + i |= i >> 2; + i |= i >> 4; + i |= i >> 8; + i |= i >> 16; + i++; + return i; +} + +struct htable *htable_alloc(uint32_t size, + htable_hash_fn_t hash_fun, htable_eq_fn_t eq_fun) +{ + struct htable *htable; + + htable = calloc(1, sizeof(*htable)); + if (!htable) { + return NULL; + } + size = round_up_to_power_of_2(size); + if (size < HTABLE_MIN_SIZE) { + size = HTABLE_MIN_SIZE; + } + htable->hash_fun = hash_fun; + htable->eq_fun = eq_fun; + htable->used = 0; + if (htable_realloc(htable, size)) { + free(htable); + return NULL; + } + return htable; +} + +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx) +{ + uint32_t i; + + for (i = 0; i != htable->capacity; ++i) { + struct htable_pair *elem = htable->elem + i; + if (elem->key) { + fun(ctx, elem->key, elem->val); + } + } +} + +void htable_free(struct htable *htable) +{ + if (htable) { + free(htable->elem); + free(htable); + } +} + +int htable_put(struct htable *htable, void *key, void *val) +{ + int ret; + uint32_t nused; + + // NULL is not a valid key value. + // This helps us implement htable_get_internal efficiently, since we know + // that we can stop when we encounter the first NULL key. + if (!key) { + return EINVAL; + } + // NULL is not a valid value. Otherwise the results of htable_get would + // be confusing (does a NULL return mean entry not found, or that the + // entry was found and was NULL?) + if (!val) { + return EINVAL; + } + // Re-hash if we have used more than half of the hash table + nused = htable->used + 1; + if (nused >= (htable->capacity / 2)) { + ret = htable_realloc(htable, htable->capacity * 2); + if (ret) + return ret; + } + htable_insert_internal(htable->elem, htable->capacity, + htable->hash_fun, key, val); + htable->used++; + return 0; +} + +static int htable_get_internal(const struct htable *htable, + const void *key, uint32_t *out) +{ + uint32_t start_idx, idx; + + start_idx = htable->hash_fun(key, htable->capacity); + idx = start_idx; + while (1) { + struct htable_pair *pair = htable->elem + idx; + if (!pair->key) { + // We always maintain the invariant that the entries corresponding + // to a given key are stored in a contiguous block, not separated + // by any NULLs. So if we encounter a NULL, our search is over. + return ENOENT; + } else if (htable->eq_fun(pair->key, key)) { + *out = idx; + return 0; + } + idx++; + if (idx == htable->capacity) { + idx = 0; + } + if (idx == start_idx) { + return ENOENT; + } + } +} + +void *htable_get(const struct htable *htable, const void *key) +{ + uint32_t idx; + + if (htable_get_internal(htable, key, &idx)) { + return NULL; + } + return htable->elem[idx].val; +} + +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val) +{ + uint32_t hole, i; + const void *nkey; + + if (htable_get_internal(htable, key, &hole)) { + *found_key = NULL; + *found_val = NULL; + return; + } + i = hole; + htable->used--; + // We need to maintain the compactness invariant used in + // htable_get_internal. This invariant specifies that the entries for any + // given key are never separated by NULLs (although they may be separated + // by entries for other keys.) + while (1) { + i++; + if (i == htable->capacity) { + i = 0; + } + nkey = htable->elem[i].key; + if (!nkey) { + *found_key = htable->elem[hole].key; + *found_val = htable->elem[hole].val; + htable->elem[hole].key = NULL; + htable->elem[hole].val = NULL; + return; + } else if (htable->eq_fun(key, nkey)) { + htable->elem[hole].key = htable->elem[i].key; + htable->elem[hole].val = htable->elem[i].val; + hole = i; + } + } +} + +uint32_t htable_used(const struct htable *htable) +{ + return htable->used; +} + +uint32_t htable_capacity(const struct htable *htable) +{ + return htable->capacity; +} + +uint32_t ht_hash_string(const void *str, uint32_t max) +{ + const char *s = str; + uint32_t hash = 0; + + while (*s) { + hash = (hash * 31) + *s; + s++; + } + return hash % max; +} + +int ht_compare_string(const void *a, const void *b) +{ + return strcmp(a, b) == 0; +} + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_3_0/common/htable.h b/libhdfs/hdfs_3_0/common/htable.h new file mode 100644 index 0000000..33f1229 --- /dev/null +++ b/libhdfs/hdfs_3_0/common/htable.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HADOOP_CORE_COMMON_HASH_TABLE +#define HADOOP_CORE_COMMON_HASH_TABLE + +#include +#include +#include + +#define HTABLE_MIN_SIZE 4 + +struct htable; + +/** + * An HTable hash function. + * + * @param key The key. + * @param capacity The total capacity. + * + * @return The hash slot. Must be less than the capacity. + */ +typedef uint32_t (*htable_hash_fn_t)(const void *key, uint32_t capacity); + +/** + * An HTable equality function. Compares two keys. + * + * @param a First key. + * @param b Second key. + * + * @return nonzero if the keys are equal. + */ +typedef int (*htable_eq_fn_t)(const void *a, const void *b); + +/** + * Allocate a new hash table. + * + * @param capacity The minimum suggested starting capacity. + * @param hash_fun The hash function to use in this hash table. + * @param eq_fun The equals function to use in this hash table. + * + * @return The new hash table on success; NULL on OOM. + */ +struct htable *htable_alloc(uint32_t capacity, htable_hash_fn_t hash_fun, + htable_eq_fn_t eq_fun); + +typedef void (*visitor_fn_t)(void *ctx, void *key, void *val); + +/** + * Visit all of the entries in the hash table. + * + * @param htable The hash table. + * @param fun The callback function to invoke on each key and value. + * @param ctx Context pointer to pass to the callback. + */ +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx); + +/** + * Free the hash table. + * + * It is up the calling code to ensure that the keys and values inside the + * table are de-allocated, if that is necessary. + * + * @param htable The hash table. + */ +void htable_free(struct htable *htable); + +/** + * Add an entry to the hash table. + * + * @param htable The hash table. + * @param key The key to add. This cannot be NULL. + * @param fun The value to add. This cannot be NULL. + * + * @return 0 on success; + * EEXIST if the value already exists in the table; + * ENOMEM if there is not enough memory to add the element. + * EFBIG if the hash table has too many entries to fit in 32 + * bits. + */ +int htable_put(struct htable *htable, void *key, void *val); + +/** + * Get an entry from the hash table. + * + * @param htable The hash table. + * @param key The key to find. + * + * @return NULL if there is no such entry; the entry otherwise. + */ +void *htable_get(const struct htable *htable, const void *key); + +/** + * Get an entry from the hash table and remove it. + * + * @param htable The hash table. + * @param key The key for the entry find and remove. + * @param found_key (out param) NULL if the entry was not found; the found key + * otherwise. + * @param found_val (out param) NULL if the entry was not found; the found + * value otherwise. + */ +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val); + +/** + * Get the number of entries used in the hash table. + * + * @param htable The hash table. + * + * @return The number of entries used in the hash table. + */ +uint32_t htable_used(const struct htable *htable); + +/** + * Get the capacity of the hash table. + * + * @param htable The hash table. + * + * @return The capacity of the hash table. + */ +uint32_t htable_capacity(const struct htable *htable); + +/** + * Hash a string. + * + * @param str The string. + * @param max Maximum hash value + * + * @return A number less than max. + */ +uint32_t ht_hash_string(const void *str, uint32_t max); + +/** + * Compare two strings. + * + * @param a The first string. + * @param b The second string. + * + * @return 1 if the strings are identical; 0 otherwise. + */ +int ht_compare_string(const void *a, const void *b); + +#endif + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_3_0/exception.c b/libhdfs/hdfs_3_0/exception.c new file mode 100644 index 0000000..dbaf1f6 --- /dev/null +++ b/libhdfs/hdfs_3_0/exception.c @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "java.lang.UnsupportedOperationException", + 0, + ENOTSUP, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +/** + * getExceptionUtilString: A helper function that calls 'methodName' in + * ExceptionUtils. The function 'methodName' should have a return type of a + * java String. + * + * @param env The JNI environment. + * @param exc The exception to get information for. + * @param methodName The method of ExceptionUtils to call that has a String + * return type. + * + * @return A C-type string containing the string returned by + * ExceptionUtils.'methodName', or NULL on failure. + */ +static char* getExceptionUtilString(JNIEnv *env, jthrowable exc, char *methodName) +{ + jthrowable jthr; + jvalue jVal; + jstring jStr = NULL; + char *excString = NULL; + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + methodName, "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + jStr = jVal.l; + jthr = newCStr(env, jStr, &excString); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + destroyLocalReference(env, jStr); + return excString; +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jthrowable jthr; + const char *stackTrace; + const char *rootCause; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + rootCause = getExceptionUtilString(env, exc, "getRootCauseMessage"); + stackTrace = getExceptionUtilString(env, exc, "getStackTrace"); + // Save the exception details in the thread-local state. + setTLSExceptionStrings(rootCause, stackTrace); + + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + if (!rootCause) { + fprintf(stderr, "(unable to get root cause for %s)\n", className); + } else { + fprintf(stderr, "%s", rootCause); + } + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + } + } + + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_3_0/exception.h b/libhdfs/hdfs_3_0/exception.h new file mode 100644 index 0000000..cdf93a1 --- /dev/null +++ b/libhdfs/hdfs_3_0/exception.h @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * printExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + * + * The root cause and stack trace exception strings retrieved from the last + * exception that happened on a thread are stored in the corresponding + * thread local state and are accessed by hdfsGetLastExceptionRootCause and + * hdfsGetLastExceptionStackTrace respectively. + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Store the information about the pending exception in the thread-local state + * and print it and free the jthrowable object. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/libhdfs/hdfs_3_0/hdfs.c b/libhdfs/hdfs_3_0/hdfs.c new file mode 100644 index 0000000..55fef24 --- /dev/null +++ b/libhdfs/hdfs_3_0/hdfs.c @@ -0,0 +1,3528 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics) +{ + jthrowable jthr; + jobject hedgedReadMetrics = NULL; + jvalue jVal; + struct hdfsHedgedReadMetrics *m = NULL; + int ret; + jobject jFS = (jobject)fs; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_DFS, + "getHedgedReadMetrics", + "()Lorg/apache/hadoop/hdfs/DFSHedgedReadMetrics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadMetrics: getHedgedReadMetrics failed"); + goto done; + } + hedgedReadMetrics = jVal.l; + + m = malloc(sizeof(struct hdfsHedgedReadMetrics)); + if (!m) { + ret = ENOMEM; + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics", + "getHedgedReadOps", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOps failed"); + goto done; + } + m->hedgedReadOps = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics", + "getHedgedReadWins", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadWins failed"); + goto done; + } + m->hedgedReadOpsWin = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics", + "getHedgedReadOpsInCurThread", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOpsInCurThread failed"); + goto done; + } + m->hedgedReadOpsInCurThread = jVal.j; + + *metrics = m; + m = NULL; + ret = 0; + +done: + destroyLocalReference(env, hedgedReadMetrics); + free(m); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics) +{ + free(metrics); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +int hdfsFileClearReadStatistics(hdfsFile file) +{ + jthrowable jthr; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return EINTERNAL; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "clearReadStatistics", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileClearReadStatistics: clearReadStatistics failed"); + goto done; + } + ret = 0; +done: + if (ret) { + errno = ret; + return ret; + } + return 0; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + struct hdfsStreamBuilder *bld = hdfsStreamBuilderAlloc(fs, path, flags); + if (bufferSize != 0) { + hdfsStreamBuilderSetBufferSize(bld, bufferSize); + } + if (replication != 0) { + hdfsStreamBuilderSetReplication(bld, replication); + } + if (blockSize != 0) { + hdfsStreamBuilderSetDefaultBlockSize(bld, blockSize); + } + return hdfsStreamBuilderBuild(bld); +} + +struct hdfsStreamBuilder { + hdfsFS fs; + int flags; + int32_t bufferSize; + int16_t replication; + int64_t defaultBlockSize; + char path[1]; +}; + +struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs, + const char *path, int flags) +{ + int path_len = strlen(path); + struct hdfsStreamBuilder *bld; + + // sizeof(hdfsStreamBuilder->path) includes one byte for the string + // terminator + bld = malloc(sizeof(struct hdfsStreamBuilder) + path_len); + if (!bld) { + errno = ENOMEM; + return NULL; + } + bld->fs = fs; + bld->flags = flags; + bld->bufferSize = 0; + bld->replication = 0; + bld->defaultBlockSize = 0; + memcpy(bld->path, path, path_len); + bld->path[path_len] = '\0'; + return bld; +} + +void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld) +{ + free(bld); +} + +int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld, + int32_t bufferSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->bufferSize = bufferSize; + return 0; +} + +int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld, + int16_t replication) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->replication = replication; + return 0; +} + +int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld, + int64_t defaultBlockSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->defaultBlockSize = defaultBlockSize; + return 0; +} + +static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, + int32_t bufferSize, int16_t replication, int64_t blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld) +{ + hdfsFile file = hdfsOpenFileImpl(bld->fs, bld->path, bld->flags, + bld->bufferSize, bld->replication, bld->defaultBlockSize); + int prevErrno = errno; + hdfsStreamBuilderFree(bld); + errno = prevErrno; + return file; +} + +int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) +{ + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + jobject jPath = NULL; + + JNIEnv *env = getJNIEnv(); + + if (!env) { + errno = EINTERNAL; + return -1; + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): constructNewObjectOfPath", path); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "truncate", JMETHOD2(JPARAM(HADOOP_PATH), "J", "Z"), + jPath, newlength); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): FileSystem#truncate", path); + return -1; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +int hdfsUnbufferFile(hdfsFile file) +{ + int ret; + jthrowable jthr; + JNIEnv *env = getJNIEnv(); + + if (!env) { + ret = EINTERNAL; + goto done; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = ENOTSUP; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, HADOOP_ISTRM, + "unbuffer", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + HADOOP_ISTRM "#unbuffer failed:"); + goto done; + } + ret = 0; + +done: + errno = ret; + return ret; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + const char *interface; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + interface = (file->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + const char *interface; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + interface = (f->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + errno = ret; + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + errno = 0; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + +char* hdfsGetLastExceptionRootCause() +{ + return getLastTLSExceptionRootCause(); +} + +char* hdfsGetLastExceptionStackTrace() +{ + return getLastTLSExceptionStackTrace(); +} + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_3_0.h b/libhdfs/hdfs_3_0/include/hdfs/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_3_0.h rename to libhdfs/hdfs_3_0/include/hdfs/hdfs.h diff --git a/libhdfs/hdfs_3_0/jni_helper.c b/libhdfs/hdfs_3_0/jni_helper.c new file mode 100644 index 0000000..c45d598 --- /dev/null +++ b/libhdfs/hdfs_3_0/jni_helper.c @@ -0,0 +1,666 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" +#include "platform.h" +#include "common/htable.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include + +static struct htable *gClassRefHTable = NULL; + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + jmethodID mid = 0; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jthrowable jthr = NULL; + jclass local_clazz = NULL; + jclass clazz = NULL; + int ret; + + mutexLock(&hdfsHashMutex); + if (!gClassRefHTable) { + gClassRefHTable = htable_alloc(MAX_HASH_TABLE_ELEM, ht_hash_string, + ht_compare_string); + if (!gClassRefHTable) { + jthr = newRuntimeError(env, "htable_alloc failed\n"); + goto done; + } + } + clazz = htable_get(gClassRefHTable, className); + if (clazz) { + *out = clazz; + goto done; + } + local_clazz = (*env)->FindClass(env,className); + if (!local_clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clazz = (*env)->NewGlobalRef(env, local_clazz); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + ret = htable_put(gClassRefHTable, (void*)className, clazz); + if (ret) { + jthr = newRuntimeError(env, "htable_put failed with error " + "code %d\n", ret); + goto done; + } + *out = clazz; + jthr = NULL; +done: + mutexUnlock(&hdfsHashMutex); + (*env)->DeleteLocalRef(env, local_clazz); + if (jthr && clazz) { + (*env)->DeleteGlobalRef(env, clazz); + } + return jthr; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (state) return state->env; + + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (state) { + mutexUnlock(&jvmMutex); + + // Free any stale exception strings. + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = NULL; + state->lastExceptionStackTrace = NULL; + + return state->env; + } + + /* Create a ThreadLocalState for this thread */ + state = threadLocalStorageCreate(); + if (!state) { + mutexUnlock(&jvmMutex); + fprintf(stderr, "getJNIEnv: Unable to create ThreadLocalState\n"); + return NULL; + } + if (threadLocalStorageSet(state)) { + mutexUnlock(&jvmMutex); + goto fail; + } + THREAD_LOCAL_STORAGE_SET_QUICK(state); + + state->env = getGlobalJNIEnv(); + mutexUnlock(&jvmMutex); + if (!state->env) { + goto fail; + } + return state->env; + +fail: + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + hdfsThreadDestructor(state); + return NULL; +} + +char* getLastTLSExceptionRootCause() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionRootCause; +} + +char* getLastTLSExceptionStackTrace() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionStackTrace; +} + +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return; + } + mutexUnlock(&jvmMutex); + } + + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = (char*)rootCause; + state->lastExceptionStackTrace = (char*)stackTrace; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_3_0/jni_helper.h b/libhdfs/hdfs_3_0/jni_helper.h new file mode 100644 index 0000000..e63ce53 --- /dev/null +++ b/libhdfs/hdfs_3_0/jni_helper.h @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * It gets this from the ThreadLocalState if it exists. If a ThreadLocalState + * does not exist, one will be created. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Get the last exception root cause that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The root cause as a C-string. + */ +char* getLastTLSExceptionRootCause(); + +/** + * Get the last exception stack trace that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The stack trace as a C-string. + */ +char* getLastTLSExceptionStackTrace(); + +/** setTLSExceptionStrings: Sets the 'rootCause' and 'stackTrace' in the + * ThreadLocalState if one exists for the current thread. + * + * @param rootCause A string containing the root cause of an exception. + * @param stackTrace A string containing the stack trace of an exception. + * @return None. + */ +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_3_0/os/mutexes.h b/libhdfs/hdfs_3_0/os/mutexes.h new file mode 100644 index 0000000..da30bf4 --- /dev/null +++ b/libhdfs/hdfs_3_0/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting the class reference hash table. */ +extern mutex hdfsHashMutex; + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/libhdfs/hdfs_3_0/os/posix/mutexes.c b/libhdfs/hdfs_3_0/os/posix/mutexes.c new file mode 100644 index 0000000..20dafaa --- /dev/null +++ b/libhdfs/hdfs_3_0/os/posix/mutexes.c @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +mutex jvmMutex; +pthread_mutexattr_t jvmMutexAttr; + +__attribute__((constructor)) static void init() { + pthread_mutexattr_init(&jvmMutexAttr); + pthread_mutexattr_settype(&jvmMutexAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&jvmMutex, &jvmMutexAttr); +} + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/libhdfs/hdfs_3_0/os/posix/platform.h b/libhdfs/hdfs_3_0/os/posix/platform.h new file mode 100644 index 0000000..c63bbf9 --- /dev/null +++ b/libhdfs/hdfs_3_0/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/libhdfs/hdfs_3_0/os/posix/thread.c b/libhdfs/hdfs_3_0/os/posix/thread.c new file mode 100644 index 0000000..af0c61f --- /dev/null +++ b/libhdfs/hdfs_3_0/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/libhdfs/hdfs_3_0/os/posix/thread_local_storage.c b/libhdfs/hdfs_3_0/os/posix/thread_local_storage.c new file mode 100644 index 0000000..e6b59d6 --- /dev/null +++ b/libhdfs/hdfs_3_0/os/posix/thread_local_storage.c @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + struct ThreadLocalState *state = (struct ThreadLocalState*)v; + JNIEnv *env = state->env;; + jint ret; + + /* Detach the current thread from the JVM */ + if (env) { + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } + } + + /* Free exception strings */ + if (state->lastExceptionStackTrace) free(state->lastExceptionStackTrace); + if (state->lastExceptionRootCause) free(state->lastExceptionRootCause); + + /* Free the state itself */ + free(state); +} + +struct ThreadLocalState* threadLocalStorageCreate() +{ + struct ThreadLocalState *state; + state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState)); + if (state == NULL) { + fprintf(stderr, + "threadLocalStorageSet: OOM - Unable to allocate thread local state\n"); + return NULL; + } + state->lastExceptionStackTrace = NULL; + state->lastExceptionRootCause = NULL; + return state; +} + +int threadLocalStorageGet(struct ThreadLocalState **state) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *state = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(struct ThreadLocalState *state) +{ + int ret = pthread_setspecific(gTlsKey, state); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(state); + } + return ret; +} diff --git a/libhdfs/hdfs_3_0/os/thread.h b/libhdfs/hdfs_3_0/os/thread.h new file mode 100644 index 0000000..ae425d3 --- /dev/null +++ b/libhdfs/hdfs_3_0/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/libhdfs/hdfs_3_0/os/thread_local_storage.h b/libhdfs/hdfs_3_0/os/thread_local_storage.h new file mode 100644 index 0000000..025ceff --- /dev/null +++ b/libhdfs/hdfs_3_0/os/thread_local_storage.h @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) \ + static __thread struct ThreadLocalState *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + *state = quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) \ + { \ + quickTlsEnv = (state); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) +#endif + +struct ThreadLocalState { + /* The JNIEnv associated with the current thread */ + JNIEnv *env; + /* The last exception stack trace that occured on this thread */ + char *lastExceptionStackTrace; + /* The last exception root cause that occured on this thread */ + char *lastExceptionRootCause; +}; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v); + +/** + * Creates an object of ThreadLocalState. + * + * @return The newly created object if successful, NULL otherwise. + */ +struct ThreadLocalState* threadLocalStorageCreate(); + +/** + * Gets the ThreadLocalState in thread-local storage for the current thread. + * If the call succeeds, and there is a ThreadLocalState associated with this + * thread, then returns 0 and populates 'state'. If the call succeeds, but + * there is no ThreadLocalState associated with this thread, then returns 0 + * and sets ThreadLocalState to NULL. If the call fails, then returns non-zero. + * Only one thread at a time may execute this function. The caller is + * responsible for enforcing mutual exclusion. + * + * @param env ThreadLocalState out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(struct ThreadLocalState **state); + +/** + * Sets the ThreadLocalState in thread-local storage for the current thread. + * + * @param env ThreadLocalState to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(struct ThreadLocalState *state); + +#endif diff --git a/libhdfs/hdfs_3_0/os/windows/inttypes.h b/libhdfs/hdfs_3_0/os/windows/inttypes.h new file mode 100644 index 0000000..a520d15 --- /dev/null +++ b/libhdfs/hdfs_3_0/os/windows/inttypes.h @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_INTTYPES_H +#define LIBHDFS_INTTYPES_H + +/* On Windows, inttypes.h does not exist, so manually define what we need. */ + +#define PRId64 "I64d" +#define PRIu64 "I64u" +typedef unsigned __int64 uint64_t; + +#endif diff --git a/libhdfs/hdfs_3_0/os/windows/mutexes.c b/libhdfs/hdfs_3_0/os/windows/mutexes.c new file mode 100644 index 0000000..875f033 --- /dev/null +++ b/libhdfs/hdfs_3_0/os/windows/mutexes.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include + +mutex hdfsHashMutex; +mutex jvmMutex; + +/** + * Unfortunately, there is no simple static initializer for a critical section. + * Instead, the API requires calling InitializeCriticalSection. Since libhdfs + * lacks an explicit initialization function, there is no obvious existing place + * for the InitializeCriticalSection calls. To work around this, we define an + * initialization function and instruct the linker to set a pointer to that + * function as a user-defined global initializer. See discussion of CRT + * Initialization: + * http://msdn.microsoft.com/en-us/library/bb918180.aspx + */ +static void __cdecl initializeMutexes(void) { + InitializeCriticalSection(&hdfsHashMutex); + InitializeCriticalSection(&jvmMutex); +} +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) +const void (__cdecl *pInitialize)(void) = initializeMutexes; + +int mutexLock(mutex *m) { + EnterCriticalSection(m); + return 0; +} + +int mutexUnlock(mutex *m) { + LeaveCriticalSection(m); + return 0; +} diff --git a/libhdfs/hdfs_3_0/os/windows/platform.h b/libhdfs/hdfs_3_0/os/windows/platform.h new file mode 100644 index 0000000..9eedfde --- /dev/null +++ b/libhdfs/hdfs_3_0/os/windows/platform.h @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include +#include +#include + +/* + * O_ACCMODE defined to match Linux definition. + */ +#ifndef O_ACCMODE +#define O_ACCMODE 0x0003 +#endif + +/* + * Windows has a different name for its maximum path length constant. + */ +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +/* + * Windows does not define EDQUOT and ESTALE in errno.h. The closest equivalents + * are these constants from winsock.h. + */ +#ifndef EDQUOT +#define EDQUOT WSAEDQUOT +#endif + +#ifndef ESTALE +#define ESTALE WSAESTALE +#endif + +/* + * gcc-style type-checked format arguments are not supported on Windows, so just + * stub this macro. + */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) + +/* + * Define macros for various string formatting functions not defined on Windows. + * Where possible, we reroute to one of the secure CRT variants. On Windows, + * the preprocessor does support variadic macros, even though they weren't + * defined until C99. + */ +#define snprintf(str, size, format, ...) \ + _snprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) +#define strncpy(dest, src, n) \ + strncpy_s((dest), (n), (src), _TRUNCATE) +#define strtok_r(str, delim, saveptr) \ + strtok_s((str), (delim), (saveptr)) +#define vsnprintf(str, size, format, ...) \ + vsnprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) + +/* + * Mutex data type defined as Windows CRITICAL_SECTION. A critical section (not + * Windows mutex) is used, because libhdfs only needs synchronization of multiple + * threads within a single process, not synchronization across process + * boundaries. + */ +typedef CRITICAL_SECTION mutex; + +/* + * Thread data type defined as HANDLE to a Windows thread. + */ +typedef HANDLE threadId; + +#endif diff --git a/libhdfs/hdfs_3_0/os/windows/thread.c b/libhdfs/hdfs_3_0/os/windows/thread.c new file mode 100644 index 0000000..f5cc2a7 --- /dev/null +++ b/libhdfs/hdfs_3_0/os/windows/thread.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by CreateThread. + * + * @param toRun thread to run + * @return DWORD result of running thread (always 0) + */ +static DWORD WINAPI runThread(LPVOID toRun) { + const thread *t = toRun; + t->start(t->arg); + return 0; +} + +int threadCreate(thread *t) { + DWORD ret = 0; + HANDLE h; + h = CreateThread(NULL, 0, runThread, t, 0, NULL); + if (h) { + t->id = h; + } else { + ret = GetLastError(); + fprintf(stderr, "threadCreate: CreateThread failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + DWORD ret = WaitForSingleObject(t->id, INFINITE); + switch (ret) { + case WAIT_OBJECT_0: + break; + case WAIT_FAILED: + ret = GetLastError(); + fprintf(stderr, "threadJoin: WaitForSingleObject failed with error %d\n", + ret); + break; + default: + fprintf(stderr, "threadJoin: WaitForSingleObject unexpected error %d\n", + ret); + break; + } + return ret; +} diff --git a/libhdfs/hdfs_3_0/os/windows/thread_local_storage.c b/libhdfs/hdfs_3_0/os/windows/thread_local_storage.c new file mode 100644 index 0000000..28d014d --- /dev/null +++ b/libhdfs/hdfs_3_0/os/windows/thread_local_storage.c @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static DWORD gTlsIndex = TLS_OUT_OF_INDEXES; + +/** + * If the current thread has a JNIEnv in thread-local storage, then detaches the + * current thread from the JVM and also frees up the ThreadLocalState object. + */ +static void detachCurrentThreadFromJvm() +{ + struct ThreadLocalState *state = NULL; + JNIEnv *env = NULL; + JavaVM *vm; + jint ret; + if (threadLocalStorageGet(&state) || !state) { + return; + } + if (!state->env) { + return; + } + env = state->env; + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, + "detachCurrentThreadFromJvm: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } + + /* Free exception strings */ + if (state->lastExceptionStackTrace) free(state->lastExceptionStackTrace); + if (state->lastExceptionRootCause) free(state->lastExceptionRootCause); + + /* Free the state itself */ + free(state); +} + +void hdfsThreadDestructor(void *v) +{ + // Ignore 'v' since it will contain the state and we will obtain it in the below + // call anyway. + detachCurrentThreadFromJvm(); +} + +/** + * Unlike pthreads, the Windows API does not seem to provide a convenient way to + * hook a callback onto thread shutdown. However, the Windows portable + * executable format does define a concept of thread-local storage callbacks. + * Here, we define a function and instruct the linker to set a pointer to that + * function in the segment for thread-local storage callbacks. See page 85 of + * Microsoft Portable Executable and Common Object File Format Specification: + * http://msdn.microsoft.com/en-us/gg463119.aspx + * This technique only works for implicit linking (OS loads DLL on demand), not + * for explicit linking (user code calls LoadLibrary directly). This effectively + * means that we have a known limitation: libhdfs may not work correctly if a + * Windows application attempts to use it via explicit linking. + * + * @param h module handle + * @param reason the reason for calling the callback + * @param pv reserved, unused + */ +static void NTAPI tlsCallback(PVOID h, DWORD reason, PVOID pv) +{ + DWORD tlsIndex; + switch (reason) { + case DLL_THREAD_DETACH: + detachCurrentThreadFromJvm(); + break; + case DLL_PROCESS_DETACH: + detachCurrentThreadFromJvm(); + tlsIndex = gTlsIndex; + gTlsIndex = TLS_OUT_OF_INDEXES; + if (!TlsFree(tlsIndex)) { + fprintf(stderr, "tlsCallback: TlsFree failed with error %d\n", + GetLastError()); + } + break; + default: + break; + } +} + +/* + * A variable named _tls_used contains the TLS directory, which contains a list + * of pointers to callback functions. Normally, the linker won't retain this + * variable unless the executable has implicit thread-local variables, defined + * using the __declspec(thread) extended storage-class modifier. libhdfs + * doesn't use __declspec(thread), and we have no guarantee that the executable + * linked to libhdfs will use __declspec(thread). By forcing the linker to + * reference _tls_used, we guarantee that the binary retains the TLS directory. + * See Microsoft Visual Studio 10.0/VC/crt/src/tlssup.c . + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#endif + +/* + * We must retain a pointer to the callback function. Force the linker to keep + * this symbol, even though it appears that nothing in our source code uses it. + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:pTlsCallback") +#else +#pragma comment(linker, "/INCLUDE:_pTlsCallback") +#endif + +/* + * Define constant pointer to our callback, and tell the linker to pin it into + * the TLS directory so that it receives thread callbacks. Use external linkage + * to protect against the linker discarding the seemingly unused symbol. + */ +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK pTlsCallback; +const PIMAGE_TLS_CALLBACK pTlsCallback = tlsCallback; +#pragma const_seg() + +struct ThreadLocalState* threadLocalStorageCreate() +{ + struct ThreadLocalState *state; + state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState)); + if (state == NULL) { + fprintf(stderr, + "threadLocalStorageSet: OOM - Unable to allocate thread local state\n"); + return NULL; + } + state->lastExceptionStackTrace = NULL; + state->lastExceptionRootCause = NULL; + return state; +} + +int threadLocalStorageGet(struct ThreadLocalState **state) +{ + LPVOID tls; + DWORD ret; + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + gTlsIndex = TlsAlloc(); + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + fprintf(stderr, + "threadLocalStorageGet: TlsAlloc failed with error %d\n", + TLS_OUT_OF_INDEXES); + return TLS_OUT_OF_INDEXES; + } + } + tls = TlsGetValue(gTlsIndex); + if (tls) { + *state = tls; + return 0; + } else { + ret = GetLastError(); + if (ERROR_SUCCESS == ret) { + /* Thread-local storage contains NULL, because we haven't set it yet. */ + *state = NULL; + return 0; + } else { + /* + * The API call failed. According to documentation, TlsGetValue cannot + * fail as long as the index is a valid index from a successful TlsAlloc + * call. This error handling is purely defensive. + */ + fprintf(stderr, + "threadLocalStorageGet: TlsGetValue failed with error %d\n", ret); + return ret; + } + } +} + +int threadLocalStorageSet(struct ThreadLocalState *state) +{ + DWORD ret = 0; + if (!TlsSetValue(gTlsIndex, (LPVOID)state)) { + ret = GetLastError(); + fprintf(stderr, + "threadLocalStorageSet: TlsSetValue failed with error %d\n", + ret); + detachCurrentThreadFromJvm(state); + } + return ret; +} diff --git a/libhdfs/hdfs_3_0/os/windows/unistd.h b/libhdfs/hdfs_3_0/os/windows/unistd.h new file mode 100644 index 0000000..b82ce48 --- /dev/null +++ b/libhdfs/hdfs_3_0/os/windows/unistd.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_UNISTD_H +#define LIBHDFS_UNISTD_H + +/* On Windows, unistd.h does not exist, so manually define what we need. */ + +#include /* Declares getpid(). */ +#include + +/* Re-route sleep to Sleep, converting units from seconds to milliseconds. */ +#define sleep(seconds) Sleep((seconds) * 1000) +#endif diff --git a/libhdfs/hdfs_3_1/common/htable.c b/libhdfs/hdfs_3_1/common/htable.c new file mode 100644 index 0000000..50c89ea --- /dev/null +++ b/libhdfs/hdfs_3_1/common/htable.c @@ -0,0 +1,287 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/htable.h" + +#include +#include +#include +#include +#include + +struct htable_pair { + void *key; + void *val; +}; + +/** + * A hash table which uses linear probing. + */ +struct htable { + uint32_t capacity; + uint32_t used; + htable_hash_fn_t hash_fun; + htable_eq_fn_t eq_fun; + struct htable_pair *elem; +}; + +/** + * An internal function for inserting a value into the hash table. + * + * Note: this function assumes that you have made enough space in the table. + * + * @param nelem The new element to insert. + * @param capacity The capacity of the hash table. + * @param hash_fun The hash function to use. + * @param key The key to insert. + * @param val The value to insert. + */ +static void htable_insert_internal(struct htable_pair *nelem, + uint32_t capacity, htable_hash_fn_t hash_fun, void *key, + void *val) +{ + uint32_t i; + + i = hash_fun(key, capacity); + while (1) { + if (!nelem[i].key) { + nelem[i].key = key; + nelem[i].val = val; + return; + } + i++; + if (i == capacity) { + i = 0; + } + } +} + +static int htable_realloc(struct htable *htable, uint32_t new_capacity) +{ + struct htable_pair *nelem; + uint32_t i, old_capacity = htable->capacity; + htable_hash_fn_t hash_fun = htable->hash_fun; + + nelem = calloc(new_capacity, sizeof(struct htable_pair)); + if (!nelem) { + return ENOMEM; + } + for (i = 0; i < old_capacity; i++) { + struct htable_pair *pair = htable->elem + i; + if (pair->key) { + htable_insert_internal(nelem, new_capacity, hash_fun, + pair->key, pair->val); + } + } + free(htable->elem); + htable->elem = nelem; + htable->capacity = new_capacity; + return 0; +} + +static uint32_t round_up_to_power_of_2(uint32_t i) +{ + if (i == 0) { + return 1; + } + i--; + i |= i >> 1; + i |= i >> 2; + i |= i >> 4; + i |= i >> 8; + i |= i >> 16; + i++; + return i; +} + +struct htable *htable_alloc(uint32_t size, + htable_hash_fn_t hash_fun, htable_eq_fn_t eq_fun) +{ + struct htable *htable; + + htable = calloc(1, sizeof(*htable)); + if (!htable) { + return NULL; + } + size = round_up_to_power_of_2(size); + if (size < HTABLE_MIN_SIZE) { + size = HTABLE_MIN_SIZE; + } + htable->hash_fun = hash_fun; + htable->eq_fun = eq_fun; + htable->used = 0; + if (htable_realloc(htable, size)) { + free(htable); + return NULL; + } + return htable; +} + +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx) +{ + uint32_t i; + + for (i = 0; i != htable->capacity; ++i) { + struct htable_pair *elem = htable->elem + i; + if (elem->key) { + fun(ctx, elem->key, elem->val); + } + } +} + +void htable_free(struct htable *htable) +{ + if (htable) { + free(htable->elem); + free(htable); + } +} + +int htable_put(struct htable *htable, void *key, void *val) +{ + int ret; + uint32_t nused; + + // NULL is not a valid key value. + // This helps us implement htable_get_internal efficiently, since we know + // that we can stop when we encounter the first NULL key. + if (!key) { + return EINVAL; + } + // NULL is not a valid value. Otherwise the results of htable_get would + // be confusing (does a NULL return mean entry not found, or that the + // entry was found and was NULL?) + if (!val) { + return EINVAL; + } + // Re-hash if we have used more than half of the hash table + nused = htable->used + 1; + if (nused >= (htable->capacity / 2)) { + ret = htable_realloc(htable, htable->capacity * 2); + if (ret) + return ret; + } + htable_insert_internal(htable->elem, htable->capacity, + htable->hash_fun, key, val); + htable->used++; + return 0; +} + +static int htable_get_internal(const struct htable *htable, + const void *key, uint32_t *out) +{ + uint32_t start_idx, idx; + + start_idx = htable->hash_fun(key, htable->capacity); + idx = start_idx; + while (1) { + struct htable_pair *pair = htable->elem + idx; + if (!pair->key) { + // We always maintain the invariant that the entries corresponding + // to a given key are stored in a contiguous block, not separated + // by any NULLs. So if we encounter a NULL, our search is over. + return ENOENT; + } else if (htable->eq_fun(pair->key, key)) { + *out = idx; + return 0; + } + idx++; + if (idx == htable->capacity) { + idx = 0; + } + if (idx == start_idx) { + return ENOENT; + } + } +} + +void *htable_get(const struct htable *htable, const void *key) +{ + uint32_t idx; + + if (htable_get_internal(htable, key, &idx)) { + return NULL; + } + return htable->elem[idx].val; +} + +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val) +{ + uint32_t hole, i; + const void *nkey; + + if (htable_get_internal(htable, key, &hole)) { + *found_key = NULL; + *found_val = NULL; + return; + } + i = hole; + htable->used--; + // We need to maintain the compactness invariant used in + // htable_get_internal. This invariant specifies that the entries for any + // given key are never separated by NULLs (although they may be separated + // by entries for other keys.) + while (1) { + i++; + if (i == htable->capacity) { + i = 0; + } + nkey = htable->elem[i].key; + if (!nkey) { + *found_key = htable->elem[hole].key; + *found_val = htable->elem[hole].val; + htable->elem[hole].key = NULL; + htable->elem[hole].val = NULL; + return; + } else if (htable->eq_fun(key, nkey)) { + htable->elem[hole].key = htable->elem[i].key; + htable->elem[hole].val = htable->elem[i].val; + hole = i; + } + } +} + +uint32_t htable_used(const struct htable *htable) +{ + return htable->used; +} + +uint32_t htable_capacity(const struct htable *htable) +{ + return htable->capacity; +} + +uint32_t ht_hash_string(const void *str, uint32_t max) +{ + const char *s = str; + uint32_t hash = 0; + + while (*s) { + hash = (hash * 31) + *s; + s++; + } + return hash % max; +} + +int ht_compare_string(const void *a, const void *b) +{ + return strcmp(a, b) == 0; +} + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_3_1/common/htable.h b/libhdfs/hdfs_3_1/common/htable.h new file mode 100644 index 0000000..33f1229 --- /dev/null +++ b/libhdfs/hdfs_3_1/common/htable.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HADOOP_CORE_COMMON_HASH_TABLE +#define HADOOP_CORE_COMMON_HASH_TABLE + +#include +#include +#include + +#define HTABLE_MIN_SIZE 4 + +struct htable; + +/** + * An HTable hash function. + * + * @param key The key. + * @param capacity The total capacity. + * + * @return The hash slot. Must be less than the capacity. + */ +typedef uint32_t (*htable_hash_fn_t)(const void *key, uint32_t capacity); + +/** + * An HTable equality function. Compares two keys. + * + * @param a First key. + * @param b Second key. + * + * @return nonzero if the keys are equal. + */ +typedef int (*htable_eq_fn_t)(const void *a, const void *b); + +/** + * Allocate a new hash table. + * + * @param capacity The minimum suggested starting capacity. + * @param hash_fun The hash function to use in this hash table. + * @param eq_fun The equals function to use in this hash table. + * + * @return The new hash table on success; NULL on OOM. + */ +struct htable *htable_alloc(uint32_t capacity, htable_hash_fn_t hash_fun, + htable_eq_fn_t eq_fun); + +typedef void (*visitor_fn_t)(void *ctx, void *key, void *val); + +/** + * Visit all of the entries in the hash table. + * + * @param htable The hash table. + * @param fun The callback function to invoke on each key and value. + * @param ctx Context pointer to pass to the callback. + */ +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx); + +/** + * Free the hash table. + * + * It is up the calling code to ensure that the keys and values inside the + * table are de-allocated, if that is necessary. + * + * @param htable The hash table. + */ +void htable_free(struct htable *htable); + +/** + * Add an entry to the hash table. + * + * @param htable The hash table. + * @param key The key to add. This cannot be NULL. + * @param fun The value to add. This cannot be NULL. + * + * @return 0 on success; + * EEXIST if the value already exists in the table; + * ENOMEM if there is not enough memory to add the element. + * EFBIG if the hash table has too many entries to fit in 32 + * bits. + */ +int htable_put(struct htable *htable, void *key, void *val); + +/** + * Get an entry from the hash table. + * + * @param htable The hash table. + * @param key The key to find. + * + * @return NULL if there is no such entry; the entry otherwise. + */ +void *htable_get(const struct htable *htable, const void *key); + +/** + * Get an entry from the hash table and remove it. + * + * @param htable The hash table. + * @param key The key for the entry find and remove. + * @param found_key (out param) NULL if the entry was not found; the found key + * otherwise. + * @param found_val (out param) NULL if the entry was not found; the found + * value otherwise. + */ +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val); + +/** + * Get the number of entries used in the hash table. + * + * @param htable The hash table. + * + * @return The number of entries used in the hash table. + */ +uint32_t htable_used(const struct htable *htable); + +/** + * Get the capacity of the hash table. + * + * @param htable The hash table. + * + * @return The capacity of the hash table. + */ +uint32_t htable_capacity(const struct htable *htable); + +/** + * Hash a string. + * + * @param str The string. + * @param max Maximum hash value + * + * @return A number less than max. + */ +uint32_t ht_hash_string(const void *str, uint32_t max); + +/** + * Compare two strings. + * + * @param a The first string. + * @param b The second string. + * + * @return 1 if the strings are identical; 0 otherwise. + */ +int ht_compare_string(const void *a, const void *b); + +#endif + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_3_1/exception.c b/libhdfs/hdfs_3_1/exception.c new file mode 100644 index 0000000..dbaf1f6 --- /dev/null +++ b/libhdfs/hdfs_3_1/exception.c @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "java.lang.UnsupportedOperationException", + 0, + ENOTSUP, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +/** + * getExceptionUtilString: A helper function that calls 'methodName' in + * ExceptionUtils. The function 'methodName' should have a return type of a + * java String. + * + * @param env The JNI environment. + * @param exc The exception to get information for. + * @param methodName The method of ExceptionUtils to call that has a String + * return type. + * + * @return A C-type string containing the string returned by + * ExceptionUtils.'methodName', or NULL on failure. + */ +static char* getExceptionUtilString(JNIEnv *env, jthrowable exc, char *methodName) +{ + jthrowable jthr; + jvalue jVal; + jstring jStr = NULL; + char *excString = NULL; + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + methodName, "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + jStr = jVal.l; + jthr = newCStr(env, jStr, &excString); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + destroyLocalReference(env, jStr); + return excString; +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jthrowable jthr; + const char *stackTrace; + const char *rootCause; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + rootCause = getExceptionUtilString(env, exc, "getRootCauseMessage"); + stackTrace = getExceptionUtilString(env, exc, "getStackTrace"); + // Save the exception details in the thread-local state. + setTLSExceptionStrings(rootCause, stackTrace); + + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + if (!rootCause) { + fprintf(stderr, "(unable to get root cause for %s)\n", className); + } else { + fprintf(stderr, "%s", rootCause); + } + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + } + } + + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_3_1/exception.h b/libhdfs/hdfs_3_1/exception.h new file mode 100644 index 0000000..cdf93a1 --- /dev/null +++ b/libhdfs/hdfs_3_1/exception.h @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * printExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + * + * The root cause and stack trace exception strings retrieved from the last + * exception that happened on a thread are stored in the corresponding + * thread local state and are accessed by hdfsGetLastExceptionRootCause and + * hdfsGetLastExceptionStackTrace respectively. + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Store the information about the pending exception in the thread-local state + * and print it and free the jthrowable object. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/libhdfs/hdfs_3_1/hdfs.c b/libhdfs/hdfs_3_1/hdfs.c new file mode 100644 index 0000000..55fef24 --- /dev/null +++ b/libhdfs/hdfs_3_1/hdfs.c @@ -0,0 +1,3528 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics) +{ + jthrowable jthr; + jobject hedgedReadMetrics = NULL; + jvalue jVal; + struct hdfsHedgedReadMetrics *m = NULL; + int ret; + jobject jFS = (jobject)fs; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_DFS, + "getHedgedReadMetrics", + "()Lorg/apache/hadoop/hdfs/DFSHedgedReadMetrics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadMetrics: getHedgedReadMetrics failed"); + goto done; + } + hedgedReadMetrics = jVal.l; + + m = malloc(sizeof(struct hdfsHedgedReadMetrics)); + if (!m) { + ret = ENOMEM; + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics", + "getHedgedReadOps", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOps failed"); + goto done; + } + m->hedgedReadOps = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics", + "getHedgedReadWins", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadWins failed"); + goto done; + } + m->hedgedReadOpsWin = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics", + "getHedgedReadOpsInCurThread", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOpsInCurThread failed"); + goto done; + } + m->hedgedReadOpsInCurThread = jVal.j; + + *metrics = m; + m = NULL; + ret = 0; + +done: + destroyLocalReference(env, hedgedReadMetrics); + free(m); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics) +{ + free(metrics); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +int hdfsFileClearReadStatistics(hdfsFile file) +{ + jthrowable jthr; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return EINTERNAL; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "clearReadStatistics", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileClearReadStatistics: clearReadStatistics failed"); + goto done; + } + ret = 0; +done: + if (ret) { + errno = ret; + return ret; + } + return 0; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + struct hdfsStreamBuilder *bld = hdfsStreamBuilderAlloc(fs, path, flags); + if (bufferSize != 0) { + hdfsStreamBuilderSetBufferSize(bld, bufferSize); + } + if (replication != 0) { + hdfsStreamBuilderSetReplication(bld, replication); + } + if (blockSize != 0) { + hdfsStreamBuilderSetDefaultBlockSize(bld, blockSize); + } + return hdfsStreamBuilderBuild(bld); +} + +struct hdfsStreamBuilder { + hdfsFS fs; + int flags; + int32_t bufferSize; + int16_t replication; + int64_t defaultBlockSize; + char path[1]; +}; + +struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs, + const char *path, int flags) +{ + int path_len = strlen(path); + struct hdfsStreamBuilder *bld; + + // sizeof(hdfsStreamBuilder->path) includes one byte for the string + // terminator + bld = malloc(sizeof(struct hdfsStreamBuilder) + path_len); + if (!bld) { + errno = ENOMEM; + return NULL; + } + bld->fs = fs; + bld->flags = flags; + bld->bufferSize = 0; + bld->replication = 0; + bld->defaultBlockSize = 0; + memcpy(bld->path, path, path_len); + bld->path[path_len] = '\0'; + return bld; +} + +void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld) +{ + free(bld); +} + +int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld, + int32_t bufferSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->bufferSize = bufferSize; + return 0; +} + +int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld, + int16_t replication) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->replication = replication; + return 0; +} + +int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld, + int64_t defaultBlockSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->defaultBlockSize = defaultBlockSize; + return 0; +} + +static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, + int32_t bufferSize, int16_t replication, int64_t blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld) +{ + hdfsFile file = hdfsOpenFileImpl(bld->fs, bld->path, bld->flags, + bld->bufferSize, bld->replication, bld->defaultBlockSize); + int prevErrno = errno; + hdfsStreamBuilderFree(bld); + errno = prevErrno; + return file; +} + +int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) +{ + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + jobject jPath = NULL; + + JNIEnv *env = getJNIEnv(); + + if (!env) { + errno = EINTERNAL; + return -1; + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): constructNewObjectOfPath", path); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "truncate", JMETHOD2(JPARAM(HADOOP_PATH), "J", "Z"), + jPath, newlength); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): FileSystem#truncate", path); + return -1; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +int hdfsUnbufferFile(hdfsFile file) +{ + int ret; + jthrowable jthr; + JNIEnv *env = getJNIEnv(); + + if (!env) { + ret = EINTERNAL; + goto done; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = ENOTSUP; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, HADOOP_ISTRM, + "unbuffer", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + HADOOP_ISTRM "#unbuffer failed:"); + goto done; + } + ret = 0; + +done: + errno = ret; + return ret; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + const char *interface; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + interface = (file->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + const char *interface; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + interface = (f->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + errno = ret; + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + errno = 0; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + +char* hdfsGetLastExceptionRootCause() +{ + return getLastTLSExceptionRootCause(); +} + +char* hdfsGetLastExceptionStackTrace() +{ + return getLastTLSExceptionStackTrace(); +} + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_3_1.h b/libhdfs/hdfs_3_1/include/hdfs/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_3_1.h rename to libhdfs/hdfs_3_1/include/hdfs/hdfs.h diff --git a/libhdfs/hdfs_3_1/jni_helper.c b/libhdfs/hdfs_3_1/jni_helper.c new file mode 100644 index 0000000..c45d598 --- /dev/null +++ b/libhdfs/hdfs_3_1/jni_helper.c @@ -0,0 +1,666 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" +#include "platform.h" +#include "common/htable.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include + +static struct htable *gClassRefHTable = NULL; + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + jmethodID mid = 0; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jthrowable jthr = NULL; + jclass local_clazz = NULL; + jclass clazz = NULL; + int ret; + + mutexLock(&hdfsHashMutex); + if (!gClassRefHTable) { + gClassRefHTable = htable_alloc(MAX_HASH_TABLE_ELEM, ht_hash_string, + ht_compare_string); + if (!gClassRefHTable) { + jthr = newRuntimeError(env, "htable_alloc failed\n"); + goto done; + } + } + clazz = htable_get(gClassRefHTable, className); + if (clazz) { + *out = clazz; + goto done; + } + local_clazz = (*env)->FindClass(env,className); + if (!local_clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clazz = (*env)->NewGlobalRef(env, local_clazz); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + ret = htable_put(gClassRefHTable, (void*)className, clazz); + if (ret) { + jthr = newRuntimeError(env, "htable_put failed with error " + "code %d\n", ret); + goto done; + } + *out = clazz; + jthr = NULL; +done: + mutexUnlock(&hdfsHashMutex); + (*env)->DeleteLocalRef(env, local_clazz); + if (jthr && clazz) { + (*env)->DeleteGlobalRef(env, clazz); + } + return jthr; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (state) return state->env; + + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (state) { + mutexUnlock(&jvmMutex); + + // Free any stale exception strings. + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = NULL; + state->lastExceptionStackTrace = NULL; + + return state->env; + } + + /* Create a ThreadLocalState for this thread */ + state = threadLocalStorageCreate(); + if (!state) { + mutexUnlock(&jvmMutex); + fprintf(stderr, "getJNIEnv: Unable to create ThreadLocalState\n"); + return NULL; + } + if (threadLocalStorageSet(state)) { + mutexUnlock(&jvmMutex); + goto fail; + } + THREAD_LOCAL_STORAGE_SET_QUICK(state); + + state->env = getGlobalJNIEnv(); + mutexUnlock(&jvmMutex); + if (!state->env) { + goto fail; + } + return state->env; + +fail: + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + hdfsThreadDestructor(state); + return NULL; +} + +char* getLastTLSExceptionRootCause() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionRootCause; +} + +char* getLastTLSExceptionStackTrace() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionStackTrace; +} + +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return; + } + mutexUnlock(&jvmMutex); + } + + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = (char*)rootCause; + state->lastExceptionStackTrace = (char*)stackTrace; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_3_1/jni_helper.h b/libhdfs/hdfs_3_1/jni_helper.h new file mode 100644 index 0000000..e63ce53 --- /dev/null +++ b/libhdfs/hdfs_3_1/jni_helper.h @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * It gets this from the ThreadLocalState if it exists. If a ThreadLocalState + * does not exist, one will be created. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Get the last exception root cause that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The root cause as a C-string. + */ +char* getLastTLSExceptionRootCause(); + +/** + * Get the last exception stack trace that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The stack trace as a C-string. + */ +char* getLastTLSExceptionStackTrace(); + +/** setTLSExceptionStrings: Sets the 'rootCause' and 'stackTrace' in the + * ThreadLocalState if one exists for the current thread. + * + * @param rootCause A string containing the root cause of an exception. + * @param stackTrace A string containing the stack trace of an exception. + * @return None. + */ +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_3_1/os/mutexes.h b/libhdfs/hdfs_3_1/os/mutexes.h new file mode 100644 index 0000000..da30bf4 --- /dev/null +++ b/libhdfs/hdfs_3_1/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting the class reference hash table. */ +extern mutex hdfsHashMutex; + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/libhdfs/hdfs_3_1/os/posix/mutexes.c b/libhdfs/hdfs_3_1/os/posix/mutexes.c new file mode 100644 index 0000000..20dafaa --- /dev/null +++ b/libhdfs/hdfs_3_1/os/posix/mutexes.c @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +mutex jvmMutex; +pthread_mutexattr_t jvmMutexAttr; + +__attribute__((constructor)) static void init() { + pthread_mutexattr_init(&jvmMutexAttr); + pthread_mutexattr_settype(&jvmMutexAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&jvmMutex, &jvmMutexAttr); +} + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/libhdfs/hdfs_3_1/os/posix/platform.h b/libhdfs/hdfs_3_1/os/posix/platform.h new file mode 100644 index 0000000..c63bbf9 --- /dev/null +++ b/libhdfs/hdfs_3_1/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/libhdfs/hdfs_3_1/os/posix/thread.c b/libhdfs/hdfs_3_1/os/posix/thread.c new file mode 100644 index 0000000..af0c61f --- /dev/null +++ b/libhdfs/hdfs_3_1/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/libhdfs/hdfs_3_1/os/posix/thread_local_storage.c b/libhdfs/hdfs_3_1/os/posix/thread_local_storage.c new file mode 100644 index 0000000..e6b59d6 --- /dev/null +++ b/libhdfs/hdfs_3_1/os/posix/thread_local_storage.c @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + struct ThreadLocalState *state = (struct ThreadLocalState*)v; + JNIEnv *env = state->env;; + jint ret; + + /* Detach the current thread from the JVM */ + if (env) { + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } + } + + /* Free exception strings */ + if (state->lastExceptionStackTrace) free(state->lastExceptionStackTrace); + if (state->lastExceptionRootCause) free(state->lastExceptionRootCause); + + /* Free the state itself */ + free(state); +} + +struct ThreadLocalState* threadLocalStorageCreate() +{ + struct ThreadLocalState *state; + state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState)); + if (state == NULL) { + fprintf(stderr, + "threadLocalStorageSet: OOM - Unable to allocate thread local state\n"); + return NULL; + } + state->lastExceptionStackTrace = NULL; + state->lastExceptionRootCause = NULL; + return state; +} + +int threadLocalStorageGet(struct ThreadLocalState **state) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *state = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(struct ThreadLocalState *state) +{ + int ret = pthread_setspecific(gTlsKey, state); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(state); + } + return ret; +} diff --git a/libhdfs/hdfs_3_1/os/thread.h b/libhdfs/hdfs_3_1/os/thread.h new file mode 100644 index 0000000..ae425d3 --- /dev/null +++ b/libhdfs/hdfs_3_1/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/libhdfs/hdfs_3_1/os/thread_local_storage.h b/libhdfs/hdfs_3_1/os/thread_local_storage.h new file mode 100644 index 0000000..025ceff --- /dev/null +++ b/libhdfs/hdfs_3_1/os/thread_local_storage.h @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) \ + static __thread struct ThreadLocalState *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + *state = quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) \ + { \ + quickTlsEnv = (state); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) +#endif + +struct ThreadLocalState { + /* The JNIEnv associated with the current thread */ + JNIEnv *env; + /* The last exception stack trace that occured on this thread */ + char *lastExceptionStackTrace; + /* The last exception root cause that occured on this thread */ + char *lastExceptionRootCause; +}; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v); + +/** + * Creates an object of ThreadLocalState. + * + * @return The newly created object if successful, NULL otherwise. + */ +struct ThreadLocalState* threadLocalStorageCreate(); + +/** + * Gets the ThreadLocalState in thread-local storage for the current thread. + * If the call succeeds, and there is a ThreadLocalState associated with this + * thread, then returns 0 and populates 'state'. If the call succeeds, but + * there is no ThreadLocalState associated with this thread, then returns 0 + * and sets ThreadLocalState to NULL. If the call fails, then returns non-zero. + * Only one thread at a time may execute this function. The caller is + * responsible for enforcing mutual exclusion. + * + * @param env ThreadLocalState out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(struct ThreadLocalState **state); + +/** + * Sets the ThreadLocalState in thread-local storage for the current thread. + * + * @param env ThreadLocalState to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(struct ThreadLocalState *state); + +#endif diff --git a/libhdfs/hdfs_3_1/os/windows/inttypes.h b/libhdfs/hdfs_3_1/os/windows/inttypes.h new file mode 100644 index 0000000..a520d15 --- /dev/null +++ b/libhdfs/hdfs_3_1/os/windows/inttypes.h @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_INTTYPES_H +#define LIBHDFS_INTTYPES_H + +/* On Windows, inttypes.h does not exist, so manually define what we need. */ + +#define PRId64 "I64d" +#define PRIu64 "I64u" +typedef unsigned __int64 uint64_t; + +#endif diff --git a/libhdfs/hdfs_3_1/os/windows/mutexes.c b/libhdfs/hdfs_3_1/os/windows/mutexes.c new file mode 100644 index 0000000..875f033 --- /dev/null +++ b/libhdfs/hdfs_3_1/os/windows/mutexes.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include + +mutex hdfsHashMutex; +mutex jvmMutex; + +/** + * Unfortunately, there is no simple static initializer for a critical section. + * Instead, the API requires calling InitializeCriticalSection. Since libhdfs + * lacks an explicit initialization function, there is no obvious existing place + * for the InitializeCriticalSection calls. To work around this, we define an + * initialization function and instruct the linker to set a pointer to that + * function as a user-defined global initializer. See discussion of CRT + * Initialization: + * http://msdn.microsoft.com/en-us/library/bb918180.aspx + */ +static void __cdecl initializeMutexes(void) { + InitializeCriticalSection(&hdfsHashMutex); + InitializeCriticalSection(&jvmMutex); +} +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) +const void (__cdecl *pInitialize)(void) = initializeMutexes; + +int mutexLock(mutex *m) { + EnterCriticalSection(m); + return 0; +} + +int mutexUnlock(mutex *m) { + LeaveCriticalSection(m); + return 0; +} diff --git a/libhdfs/hdfs_3_1/os/windows/platform.h b/libhdfs/hdfs_3_1/os/windows/platform.h new file mode 100644 index 0000000..9eedfde --- /dev/null +++ b/libhdfs/hdfs_3_1/os/windows/platform.h @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include +#include +#include + +/* + * O_ACCMODE defined to match Linux definition. + */ +#ifndef O_ACCMODE +#define O_ACCMODE 0x0003 +#endif + +/* + * Windows has a different name for its maximum path length constant. + */ +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +/* + * Windows does not define EDQUOT and ESTALE in errno.h. The closest equivalents + * are these constants from winsock.h. + */ +#ifndef EDQUOT +#define EDQUOT WSAEDQUOT +#endif + +#ifndef ESTALE +#define ESTALE WSAESTALE +#endif + +/* + * gcc-style type-checked format arguments are not supported on Windows, so just + * stub this macro. + */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) + +/* + * Define macros for various string formatting functions not defined on Windows. + * Where possible, we reroute to one of the secure CRT variants. On Windows, + * the preprocessor does support variadic macros, even though they weren't + * defined until C99. + */ +#define snprintf(str, size, format, ...) \ + _snprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) +#define strncpy(dest, src, n) \ + strncpy_s((dest), (n), (src), _TRUNCATE) +#define strtok_r(str, delim, saveptr) \ + strtok_s((str), (delim), (saveptr)) +#define vsnprintf(str, size, format, ...) \ + vsnprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) + +/* + * Mutex data type defined as Windows CRITICAL_SECTION. A critical section (not + * Windows mutex) is used, because libhdfs only needs synchronization of multiple + * threads within a single process, not synchronization across process + * boundaries. + */ +typedef CRITICAL_SECTION mutex; + +/* + * Thread data type defined as HANDLE to a Windows thread. + */ +typedef HANDLE threadId; + +#endif diff --git a/libhdfs/hdfs_3_1/os/windows/thread.c b/libhdfs/hdfs_3_1/os/windows/thread.c new file mode 100644 index 0000000..f5cc2a7 --- /dev/null +++ b/libhdfs/hdfs_3_1/os/windows/thread.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by CreateThread. + * + * @param toRun thread to run + * @return DWORD result of running thread (always 0) + */ +static DWORD WINAPI runThread(LPVOID toRun) { + const thread *t = toRun; + t->start(t->arg); + return 0; +} + +int threadCreate(thread *t) { + DWORD ret = 0; + HANDLE h; + h = CreateThread(NULL, 0, runThread, t, 0, NULL); + if (h) { + t->id = h; + } else { + ret = GetLastError(); + fprintf(stderr, "threadCreate: CreateThread failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + DWORD ret = WaitForSingleObject(t->id, INFINITE); + switch (ret) { + case WAIT_OBJECT_0: + break; + case WAIT_FAILED: + ret = GetLastError(); + fprintf(stderr, "threadJoin: WaitForSingleObject failed with error %d\n", + ret); + break; + default: + fprintf(stderr, "threadJoin: WaitForSingleObject unexpected error %d\n", + ret); + break; + } + return ret; +} diff --git a/libhdfs/hdfs_3_1/os/windows/thread_local_storage.c b/libhdfs/hdfs_3_1/os/windows/thread_local_storage.c new file mode 100644 index 0000000..28d014d --- /dev/null +++ b/libhdfs/hdfs_3_1/os/windows/thread_local_storage.c @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static DWORD gTlsIndex = TLS_OUT_OF_INDEXES; + +/** + * If the current thread has a JNIEnv in thread-local storage, then detaches the + * current thread from the JVM and also frees up the ThreadLocalState object. + */ +static void detachCurrentThreadFromJvm() +{ + struct ThreadLocalState *state = NULL; + JNIEnv *env = NULL; + JavaVM *vm; + jint ret; + if (threadLocalStorageGet(&state) || !state) { + return; + } + if (!state->env) { + return; + } + env = state->env; + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, + "detachCurrentThreadFromJvm: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } + + /* Free exception strings */ + if (state->lastExceptionStackTrace) free(state->lastExceptionStackTrace); + if (state->lastExceptionRootCause) free(state->lastExceptionRootCause); + + /* Free the state itself */ + free(state); +} + +void hdfsThreadDestructor(void *v) +{ + // Ignore 'v' since it will contain the state and we will obtain it in the below + // call anyway. + detachCurrentThreadFromJvm(); +} + +/** + * Unlike pthreads, the Windows API does not seem to provide a convenient way to + * hook a callback onto thread shutdown. However, the Windows portable + * executable format does define a concept of thread-local storage callbacks. + * Here, we define a function and instruct the linker to set a pointer to that + * function in the segment for thread-local storage callbacks. See page 85 of + * Microsoft Portable Executable and Common Object File Format Specification: + * http://msdn.microsoft.com/en-us/gg463119.aspx + * This technique only works for implicit linking (OS loads DLL on demand), not + * for explicit linking (user code calls LoadLibrary directly). This effectively + * means that we have a known limitation: libhdfs may not work correctly if a + * Windows application attempts to use it via explicit linking. + * + * @param h module handle + * @param reason the reason for calling the callback + * @param pv reserved, unused + */ +static void NTAPI tlsCallback(PVOID h, DWORD reason, PVOID pv) +{ + DWORD tlsIndex; + switch (reason) { + case DLL_THREAD_DETACH: + detachCurrentThreadFromJvm(); + break; + case DLL_PROCESS_DETACH: + detachCurrentThreadFromJvm(); + tlsIndex = gTlsIndex; + gTlsIndex = TLS_OUT_OF_INDEXES; + if (!TlsFree(tlsIndex)) { + fprintf(stderr, "tlsCallback: TlsFree failed with error %d\n", + GetLastError()); + } + break; + default: + break; + } +} + +/* + * A variable named _tls_used contains the TLS directory, which contains a list + * of pointers to callback functions. Normally, the linker won't retain this + * variable unless the executable has implicit thread-local variables, defined + * using the __declspec(thread) extended storage-class modifier. libhdfs + * doesn't use __declspec(thread), and we have no guarantee that the executable + * linked to libhdfs will use __declspec(thread). By forcing the linker to + * reference _tls_used, we guarantee that the binary retains the TLS directory. + * See Microsoft Visual Studio 10.0/VC/crt/src/tlssup.c . + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#endif + +/* + * We must retain a pointer to the callback function. Force the linker to keep + * this symbol, even though it appears that nothing in our source code uses it. + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:pTlsCallback") +#else +#pragma comment(linker, "/INCLUDE:_pTlsCallback") +#endif + +/* + * Define constant pointer to our callback, and tell the linker to pin it into + * the TLS directory so that it receives thread callbacks. Use external linkage + * to protect against the linker discarding the seemingly unused symbol. + */ +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK pTlsCallback; +const PIMAGE_TLS_CALLBACK pTlsCallback = tlsCallback; +#pragma const_seg() + +struct ThreadLocalState* threadLocalStorageCreate() +{ + struct ThreadLocalState *state; + state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState)); + if (state == NULL) { + fprintf(stderr, + "threadLocalStorageSet: OOM - Unable to allocate thread local state\n"); + return NULL; + } + state->lastExceptionStackTrace = NULL; + state->lastExceptionRootCause = NULL; + return state; +} + +int threadLocalStorageGet(struct ThreadLocalState **state) +{ + LPVOID tls; + DWORD ret; + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + gTlsIndex = TlsAlloc(); + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + fprintf(stderr, + "threadLocalStorageGet: TlsAlloc failed with error %d\n", + TLS_OUT_OF_INDEXES); + return TLS_OUT_OF_INDEXES; + } + } + tls = TlsGetValue(gTlsIndex); + if (tls) { + *state = tls; + return 0; + } else { + ret = GetLastError(); + if (ERROR_SUCCESS == ret) { + /* Thread-local storage contains NULL, because we haven't set it yet. */ + *state = NULL; + return 0; + } else { + /* + * The API call failed. According to documentation, TlsGetValue cannot + * fail as long as the index is a valid index from a successful TlsAlloc + * call. This error handling is purely defensive. + */ + fprintf(stderr, + "threadLocalStorageGet: TlsGetValue failed with error %d\n", ret); + return ret; + } + } +} + +int threadLocalStorageSet(struct ThreadLocalState *state) +{ + DWORD ret = 0; + if (!TlsSetValue(gTlsIndex, (LPVOID)state)) { + ret = GetLastError(); + fprintf(stderr, + "threadLocalStorageSet: TlsSetValue failed with error %d\n", + ret); + detachCurrentThreadFromJvm(state); + } + return ret; +} diff --git a/libhdfs/hdfs_3_1/os/windows/unistd.h b/libhdfs/hdfs_3_1/os/windows/unistd.h new file mode 100644 index 0000000..b82ce48 --- /dev/null +++ b/libhdfs/hdfs_3_1/os/windows/unistd.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_UNISTD_H +#define LIBHDFS_UNISTD_H + +/* On Windows, unistd.h does not exist, so manually define what we need. */ + +#include /* Declares getpid(). */ +#include + +/* Re-route sleep to Sleep, converting units from seconds to milliseconds. */ +#define sleep(seconds) Sleep((seconds) * 1000) +#endif diff --git a/libhdfs/hdfs_3_2/common/htable.c b/libhdfs/hdfs_3_2/common/htable.c new file mode 100644 index 0000000..50c89ea --- /dev/null +++ b/libhdfs/hdfs_3_2/common/htable.c @@ -0,0 +1,287 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "common/htable.h" + +#include +#include +#include +#include +#include + +struct htable_pair { + void *key; + void *val; +}; + +/** + * A hash table which uses linear probing. + */ +struct htable { + uint32_t capacity; + uint32_t used; + htable_hash_fn_t hash_fun; + htable_eq_fn_t eq_fun; + struct htable_pair *elem; +}; + +/** + * An internal function for inserting a value into the hash table. + * + * Note: this function assumes that you have made enough space in the table. + * + * @param nelem The new element to insert. + * @param capacity The capacity of the hash table. + * @param hash_fun The hash function to use. + * @param key The key to insert. + * @param val The value to insert. + */ +static void htable_insert_internal(struct htable_pair *nelem, + uint32_t capacity, htable_hash_fn_t hash_fun, void *key, + void *val) +{ + uint32_t i; + + i = hash_fun(key, capacity); + while (1) { + if (!nelem[i].key) { + nelem[i].key = key; + nelem[i].val = val; + return; + } + i++; + if (i == capacity) { + i = 0; + } + } +} + +static int htable_realloc(struct htable *htable, uint32_t new_capacity) +{ + struct htable_pair *nelem; + uint32_t i, old_capacity = htable->capacity; + htable_hash_fn_t hash_fun = htable->hash_fun; + + nelem = calloc(new_capacity, sizeof(struct htable_pair)); + if (!nelem) { + return ENOMEM; + } + for (i = 0; i < old_capacity; i++) { + struct htable_pair *pair = htable->elem + i; + if (pair->key) { + htable_insert_internal(nelem, new_capacity, hash_fun, + pair->key, pair->val); + } + } + free(htable->elem); + htable->elem = nelem; + htable->capacity = new_capacity; + return 0; +} + +static uint32_t round_up_to_power_of_2(uint32_t i) +{ + if (i == 0) { + return 1; + } + i--; + i |= i >> 1; + i |= i >> 2; + i |= i >> 4; + i |= i >> 8; + i |= i >> 16; + i++; + return i; +} + +struct htable *htable_alloc(uint32_t size, + htable_hash_fn_t hash_fun, htable_eq_fn_t eq_fun) +{ + struct htable *htable; + + htable = calloc(1, sizeof(*htable)); + if (!htable) { + return NULL; + } + size = round_up_to_power_of_2(size); + if (size < HTABLE_MIN_SIZE) { + size = HTABLE_MIN_SIZE; + } + htable->hash_fun = hash_fun; + htable->eq_fun = eq_fun; + htable->used = 0; + if (htable_realloc(htable, size)) { + free(htable); + return NULL; + } + return htable; +} + +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx) +{ + uint32_t i; + + for (i = 0; i != htable->capacity; ++i) { + struct htable_pair *elem = htable->elem + i; + if (elem->key) { + fun(ctx, elem->key, elem->val); + } + } +} + +void htable_free(struct htable *htable) +{ + if (htable) { + free(htable->elem); + free(htable); + } +} + +int htable_put(struct htable *htable, void *key, void *val) +{ + int ret; + uint32_t nused; + + // NULL is not a valid key value. + // This helps us implement htable_get_internal efficiently, since we know + // that we can stop when we encounter the first NULL key. + if (!key) { + return EINVAL; + } + // NULL is not a valid value. Otherwise the results of htable_get would + // be confusing (does a NULL return mean entry not found, or that the + // entry was found and was NULL?) + if (!val) { + return EINVAL; + } + // Re-hash if we have used more than half of the hash table + nused = htable->used + 1; + if (nused >= (htable->capacity / 2)) { + ret = htable_realloc(htable, htable->capacity * 2); + if (ret) + return ret; + } + htable_insert_internal(htable->elem, htable->capacity, + htable->hash_fun, key, val); + htable->used++; + return 0; +} + +static int htable_get_internal(const struct htable *htable, + const void *key, uint32_t *out) +{ + uint32_t start_idx, idx; + + start_idx = htable->hash_fun(key, htable->capacity); + idx = start_idx; + while (1) { + struct htable_pair *pair = htable->elem + idx; + if (!pair->key) { + // We always maintain the invariant that the entries corresponding + // to a given key are stored in a contiguous block, not separated + // by any NULLs. So if we encounter a NULL, our search is over. + return ENOENT; + } else if (htable->eq_fun(pair->key, key)) { + *out = idx; + return 0; + } + idx++; + if (idx == htable->capacity) { + idx = 0; + } + if (idx == start_idx) { + return ENOENT; + } + } +} + +void *htable_get(const struct htable *htable, const void *key) +{ + uint32_t idx; + + if (htable_get_internal(htable, key, &idx)) { + return NULL; + } + return htable->elem[idx].val; +} + +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val) +{ + uint32_t hole, i; + const void *nkey; + + if (htable_get_internal(htable, key, &hole)) { + *found_key = NULL; + *found_val = NULL; + return; + } + i = hole; + htable->used--; + // We need to maintain the compactness invariant used in + // htable_get_internal. This invariant specifies that the entries for any + // given key are never separated by NULLs (although they may be separated + // by entries for other keys.) + while (1) { + i++; + if (i == htable->capacity) { + i = 0; + } + nkey = htable->elem[i].key; + if (!nkey) { + *found_key = htable->elem[hole].key; + *found_val = htable->elem[hole].val; + htable->elem[hole].key = NULL; + htable->elem[hole].val = NULL; + return; + } else if (htable->eq_fun(key, nkey)) { + htable->elem[hole].key = htable->elem[i].key; + htable->elem[hole].val = htable->elem[i].val; + hole = i; + } + } +} + +uint32_t htable_used(const struct htable *htable) +{ + return htable->used; +} + +uint32_t htable_capacity(const struct htable *htable) +{ + return htable->capacity; +} + +uint32_t ht_hash_string(const void *str, uint32_t max) +{ + const char *s = str; + uint32_t hash = 0; + + while (*s) { + hash = (hash * 31) + *s; + s++; + } + return hash % max; +} + +int ht_compare_string(const void *a, const void *b) +{ + return strcmp(a, b) == 0; +} + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_3_2/common/htable.h b/libhdfs/hdfs_3_2/common/htable.h new file mode 100644 index 0000000..33f1229 --- /dev/null +++ b/libhdfs/hdfs_3_2/common/htable.h @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HADOOP_CORE_COMMON_HASH_TABLE +#define HADOOP_CORE_COMMON_HASH_TABLE + +#include +#include +#include + +#define HTABLE_MIN_SIZE 4 + +struct htable; + +/** + * An HTable hash function. + * + * @param key The key. + * @param capacity The total capacity. + * + * @return The hash slot. Must be less than the capacity. + */ +typedef uint32_t (*htable_hash_fn_t)(const void *key, uint32_t capacity); + +/** + * An HTable equality function. Compares two keys. + * + * @param a First key. + * @param b Second key. + * + * @return nonzero if the keys are equal. + */ +typedef int (*htable_eq_fn_t)(const void *a, const void *b); + +/** + * Allocate a new hash table. + * + * @param capacity The minimum suggested starting capacity. + * @param hash_fun The hash function to use in this hash table. + * @param eq_fun The equals function to use in this hash table. + * + * @return The new hash table on success; NULL on OOM. + */ +struct htable *htable_alloc(uint32_t capacity, htable_hash_fn_t hash_fun, + htable_eq_fn_t eq_fun); + +typedef void (*visitor_fn_t)(void *ctx, void *key, void *val); + +/** + * Visit all of the entries in the hash table. + * + * @param htable The hash table. + * @param fun The callback function to invoke on each key and value. + * @param ctx Context pointer to pass to the callback. + */ +void htable_visit(struct htable *htable, visitor_fn_t fun, void *ctx); + +/** + * Free the hash table. + * + * It is up the calling code to ensure that the keys and values inside the + * table are de-allocated, if that is necessary. + * + * @param htable The hash table. + */ +void htable_free(struct htable *htable); + +/** + * Add an entry to the hash table. + * + * @param htable The hash table. + * @param key The key to add. This cannot be NULL. + * @param fun The value to add. This cannot be NULL. + * + * @return 0 on success; + * EEXIST if the value already exists in the table; + * ENOMEM if there is not enough memory to add the element. + * EFBIG if the hash table has too many entries to fit in 32 + * bits. + */ +int htable_put(struct htable *htable, void *key, void *val); + +/** + * Get an entry from the hash table. + * + * @param htable The hash table. + * @param key The key to find. + * + * @return NULL if there is no such entry; the entry otherwise. + */ +void *htable_get(const struct htable *htable, const void *key); + +/** + * Get an entry from the hash table and remove it. + * + * @param htable The hash table. + * @param key The key for the entry find and remove. + * @param found_key (out param) NULL if the entry was not found; the found key + * otherwise. + * @param found_val (out param) NULL if the entry was not found; the found + * value otherwise. + */ +void htable_pop(struct htable *htable, const void *key, + void **found_key, void **found_val); + +/** + * Get the number of entries used in the hash table. + * + * @param htable The hash table. + * + * @return The number of entries used in the hash table. + */ +uint32_t htable_used(const struct htable *htable); + +/** + * Get the capacity of the hash table. + * + * @param htable The hash table. + * + * @return The capacity of the hash table. + */ +uint32_t htable_capacity(const struct htable *htable); + +/** + * Hash a string. + * + * @param str The string. + * @param max Maximum hash value + * + * @return A number less than max. + */ +uint32_t ht_hash_string(const void *str, uint32_t max); + +/** + * Compare two strings. + * + * @param a The first string. + * @param b The second string. + * + * @return 1 if the strings are identical; 0 otherwise. + */ +int ht_compare_string(const void *a, const void *b); + +#endif + +// vim: ts=4:sw=4:tw=79:et diff --git a/libhdfs/hdfs_3_2/exception.c b/libhdfs/hdfs_3_2/exception.c new file mode 100644 index 0000000..dbaf1f6 --- /dev/null +++ b/libhdfs/hdfs_3_2/exception.c @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "java.lang.UnsupportedOperationException", + 0, + ENOTSUP, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +/** + * getExceptionUtilString: A helper function that calls 'methodName' in + * ExceptionUtils. The function 'methodName' should have a return type of a + * java String. + * + * @param env The JNI environment. + * @param exc The exception to get information for. + * @param methodName The method of ExceptionUtils to call that has a String + * return type. + * + * @return A C-type string containing the string returned by + * ExceptionUtils.'methodName', or NULL on failure. + */ +static char* getExceptionUtilString(JNIEnv *env, jthrowable exc, char *methodName) +{ + jthrowable jthr; + jvalue jVal; + jstring jStr = NULL; + char *excString = NULL; + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "org/apache/commons/lang/exception/ExceptionUtils", + methodName, "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + jStr = jVal.l; + jthr = newCStr(env, jStr, &excString); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + destroyLocalReference(env, jStr); + return excString; +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jthrowable jthr; + const char *stackTrace; + const char *rootCause; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + rootCause = getExceptionUtilString(env, exc, "getRootCauseMessage"); + stackTrace = getExceptionUtilString(env, exc, "getStackTrace"); + // Save the exception details in the thread-local state. + setTLSExceptionStrings(rootCause, stackTrace); + + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + if (!rootCause) { + fprintf(stderr, "(unable to get root cause for %s)\n", className); + } else { + fprintf(stderr, "%s", rootCause); + } + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + } + } + + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_3_2/exception.h b/libhdfs/hdfs_3_2/exception.h new file mode 100644 index 0000000..cdf93a1 --- /dev/null +++ b/libhdfs/hdfs_3_2/exception.h @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * printExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + * + * The root cause and stack trace exception strings retrieved from the last + * exception that happened on a thread are stored in the corresponding + * thread local state and are accessed by hdfsGetLastExceptionRootCause and + * hdfsGetLastExceptionStackTrace respectively. + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Store the information about the pending exception in the thread-local state + * and print it and free the jthrowable object. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/libhdfs/hdfs_3_2/hdfs.c b/libhdfs/hdfs_3_2/hdfs.c new file mode 100644 index 0000000..55fef24 --- /dev/null +++ b/libhdfs/hdfs_3_2/hdfs.c @@ -0,0 +1,3528 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +/* Some frequently used Java paths */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_ISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_OSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_STAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_STRING "java/lang/String" +#define READ_OPTION "org/apache/hadoop/fs/ReadOption" + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics) +{ + jthrowable jthr; + jobject hedgedReadMetrics = NULL; + jvalue jVal; + struct hdfsHedgedReadMetrics *m = NULL; + int ret; + jobject jFS = (jobject)fs; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_DFS, + "getHedgedReadMetrics", + "()Lorg/apache/hadoop/hdfs/DFSHedgedReadMetrics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadMetrics: getHedgedReadMetrics failed"); + goto done; + } + hedgedReadMetrics = jVal.l; + + m = malloc(sizeof(struct hdfsHedgedReadMetrics)); + if (!m) { + ret = ENOMEM; + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics", + "getHedgedReadOps", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOps failed"); + goto done; + } + m->hedgedReadOps = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics", + "getHedgedReadWins", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadWins failed"); + goto done; + } + m->hedgedReadOpsWin = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics", + "getHedgedReadOpsInCurThread", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOpsInCurThread failed"); + goto done; + } + m->hedgedReadOpsInCurThread = jVal.j; + + *metrics = m; + m = NULL; + ret = 0; + +done: + destroyLocalReference(env, hedgedReadMetrics); + free(m); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics) +{ + free(metrics); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalShortCircuitBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + "org/apache/hadoop/hdfs/ReadStatistics", + "getTotalZeroCopyBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +int hdfsFileClearReadStatistics(hdfsFile file) +{ + jthrowable jthr; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return EINTERNAL; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + "org/apache/hadoop/hdfs/client/HdfsDataInputStream", + "clearReadStatistics", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileClearReadStatistics: clearReadStatistics failed"); + goto done; + } + ret = 0; +done: + if (ret) { + errno = ret; + return ret; + } + return 0; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return !!(file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ); +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/net/unix/DomainSocket", + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfClass(env, &jPath, "org/apache/hadoop/fs/Path", + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), + jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfClass(env, &jConfiguration, HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstanceLocal", JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), + JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JAVA_NET_URI, + "create", "(Ljava/lang/String;)Ljava/net/URI;", + jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, + "newInstance", JMETHOD3(JPARAM(JAVA_NET_URI), + JPARAM(HADOOP_CONF), JPARAM(JAVA_STRING), + JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, HADOOP_FS, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), + jURI, jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + struct hdfsStreamBuilder *bld = hdfsStreamBuilderAlloc(fs, path, flags); + if (bufferSize != 0) { + hdfsStreamBuilderSetBufferSize(bld, bufferSize); + } + if (replication != 0) { + hdfsStreamBuilderSetReplication(bld, replication); + } + if (blockSize != 0) { + hdfsStreamBuilderSetDefaultBlockSize(bld, blockSize); + } + return hdfsStreamBuilderBuild(bld); +} + +struct hdfsStreamBuilder { + hdfsFS fs; + int flags; + int32_t bufferSize; + int16_t replication; + int64_t defaultBlockSize; + char path[1]; +}; + +struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs, + const char *path, int flags) +{ + int path_len = strlen(path); + struct hdfsStreamBuilder *bld; + + // sizeof(hdfsStreamBuilder->path) includes one byte for the string + // terminator + bld = malloc(sizeof(struct hdfsStreamBuilder) + path_len); + if (!bld) { + errno = ENOMEM; + return NULL; + } + bld->fs = fs; + bld->flags = flags; + bld->bufferSize = 0; + bld->replication = 0; + bld->defaultBlockSize = 0; + memcpy(bld->path, path, path_len); + bld->path[path_len] = '\0'; + return bld; +} + +void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld) +{ + free(bld); +} + +int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld, + int32_t bufferSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->bufferSize = bufferSize; + return 0; +} + +int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld, + int16_t replication) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->replication = replication; + return 0; +} + +int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld, + int64_t defaultBlockSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->defaultBlockSize = defaultBlockSize; + return 0; +} + +static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, + int32_t bufferSize, int16_t replication, int64_t blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_ISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_OSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_OSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + HADOOP_CONF, "getInt", "(Ljava/lang/String;I)I", + jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + method, signature, jPath, jOverWrite, + jBufferSize, jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Try a test read to see if we can do direct reads + char buf; + if (readDirect(fs, file, &buf, 0) == 0) { + // Success - 0-byte read should return 0 + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } else if (errno != ENOTSUP) { + // Unexpected error. Clear it, don't set the direct flag. + fprintf(stderr, + "hdfsOpenFile(%s): WARN: Unexpected error %d when testing " + "for direct read compatibility\n", path, errno); + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld) +{ + hdfsFile file = hdfsOpenFileImpl(bld->fs, bld->path, bld->flags, + bld->bufferSize, bld->replication, bld->defaultBlockSize); + int prevErrno = errno; + hdfsStreamBuilderFree(bld); + errno = prevErrno; + return file; +} + +int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) +{ + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + jobject jPath = NULL; + + JNIEnv *env = getJNIEnv(); + + if (!env) { + errno = EINTERNAL; + return -1; + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): constructNewObjectOfPath", path); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "truncate", JMETHOD2(JPARAM(HADOOP_PATH), "J", "Z"), + jPath, newlength); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): FileSystem#truncate", path); + return -1; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +int hdfsUnbufferFile(hdfsFile file) +{ + int ret; + jthrowable jthr; + JNIEnv *env = getJNIEnv(); + + if (!env) { + ret = EINTERNAL; + goto done; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = ENOTSUP; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, HADOOP_ISTRM, + "unbuffer", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + HADOOP_ISTRM "#unbuffer failed:"); + goto done; + } + ret = 0; + +done: + errno = ret; + return ret; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + const char *interface; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + interface = (file->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, interface, + "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jint noReadBytes = length; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, HADOOP_ISTRM, + "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, noReadBytes, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +// Reads using the read(ByteBuffer) API, which does fewer copies +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer bbuffer = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(bbuffer); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "read", "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + return (jVal.i < 0) ? 0 : jVal.i; +} + +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, HADOOP_ISTRM, + "read", "(J[BII)I", position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "write", "([B)V", jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + HADOOP_ISTRM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + + + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + const char *interface; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + interface = (f->type == HDFS_STREAM_INPUT) ? + HADOOP_ISTRM : HADOOP_OSTRM; + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + interface, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + HADOOP_OSTRM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + HADOOP_OSTRM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + HADOOP_ISTRM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfClass(env, &jConfiguration, + HADOOP_CONF, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, + NULL, "org/apache/hadoop/fs/FileUtil", "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", + jPath, jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, "rename", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_PATH), "Z"), + jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getWorkingDirectory", + "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, + "org/apache/hadoop/fs/Path", "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setWorkingDirectory", + "(Lorg/apache/hadoop/fs/Path;)V", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfClass(env, &jPermObj, + HADOOP_FSPERM,"(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfClass(%s)", HADOOP_FSPERM); + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setPermission", + JMETHOD2(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSPERM), JAVA_VOID), + jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, HADOOP_FS, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", JAVA_VOID), + jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + errno = EINVAL; + return -1; + } + } + if (opts->byteBufferPool) { + // Delete any previous ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = byteBufferPool; + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, READ_OPTION, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "of", + "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, READ_OPTION); + if (!clazz) { + jthr = newRuntimeError(env, "failed " + "to find class for %s", READ_OPTION); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + "java/util/EnumSet", "noneOf", + "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + "java/nio/ByteBuffer", "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, HADOOP_ISTRM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + HADOOP_ISTRM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", + jFileStatus, start, length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + if (!jFileBlock) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, HADOOP_BLK_LOC, + "getHosts", "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + if (!jHost) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + errno = ret; + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, HADOOP_FSSTATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isDir", "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, HADOOP_PATH, + "toString", "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getOwner", "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, + HADOOP_STAT, "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, HADOOP_STAT, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_STAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, HADOOP_FSPERM, + "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + destroyLocalReference(env, jPath); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), + jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + HADOOP_FS, "getFileStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_STAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_DFS, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_STAT)), + jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + if (!tmpStat) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + errno = 0; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + +char* hdfsGetLastExceptionRootCause() +{ + return getLastTLSExceptionRootCause(); +} + +char* hdfsGetLastExceptionStackTrace() +{ + return getLastTLSExceptionStackTrace(); +} + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_3_2.h b/libhdfs/hdfs_3_2/include/hdfs/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_3_2.h rename to libhdfs/hdfs_3_2/include/hdfs/hdfs.h diff --git a/libhdfs/hdfs_3_2/jni_helper.c b/libhdfs/hdfs_3_2/jni_helper.c new file mode 100644 index 0000000..c45d598 --- /dev/null +++ b/libhdfs/hdfs_3_2/jni_helper.c @@ -0,0 +1,666 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jni_helper.h" +#include "platform.h" +#include "common/htable.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include + +static struct htable *gClassRefHTable = NULL; + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + + +/** + * MAX_HASH_TABLE_ELEM: The maximum no. of entries in the hashtable. + * It's set to 4096 to account for (classNames + No. of threads) + */ +#define MAX_HASH_TABLE_ELEM 4096 + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, methName, methSignature, + methType, env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + va_start(args, methSignature); + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + va_end(args); + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = methodIdFromClass(className, "", ctorSignature, + INSTANCE, env, &mid); + if (jthr) + return jthr; + va_start(args, ctorSignature); + jobj = (*env)->NewObjectV(env, cls, mid, args); + va_end(args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jclass cls; + jthrowable jthr; + jmethodID mid = 0; + + jthr = globalClassReference(className, env, &cls); + if (jthr) + return jthr; + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out) +{ + jthrowable jthr = NULL; + jclass local_clazz = NULL; + jclass clazz = NULL; + int ret; + + mutexLock(&hdfsHashMutex); + if (!gClassRefHTable) { + gClassRefHTable = htable_alloc(MAX_HASH_TABLE_ELEM, ht_hash_string, + ht_compare_string); + if (!gClassRefHTable) { + jthr = newRuntimeError(env, "htable_alloc failed\n"); + goto done; + } + } + clazz = htable_get(gClassRefHTable, className); + if (clazz) { + *out = clazz; + goto done; + } + local_clazz = (*env)->FindClass(env,className); + if (!local_clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clazz = (*env)->NewGlobalRef(env, local_clazz); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + ret = htable_put(gClassRefHTable, (void*)className, clazz); + if (ret) { + jthr = newRuntimeError(env, "htable_put failed with error " + "code %d\n", ret); + goto done; + } + *out = clazz; + jthr = NULL; +done: + mutexUnlock(&hdfsHashMutex); + (*env)->DeleteLocalRef(env, local_clazz); + if (jthr && clazz) { + (*env)->DeleteGlobalRef(env, clazz); + } + return jthr; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getenv("CLASSPATH"); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, + "org/apache/hadoop/fs/FileSystem", + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "loadFileSystems"); + } + } + else { + //Attach this thread to the VM + vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (state) return state->env; + + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (state) { + mutexUnlock(&jvmMutex); + + // Free any stale exception strings. + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = NULL; + state->lastExceptionStackTrace = NULL; + + return state->env; + } + + /* Create a ThreadLocalState for this thread */ + state = threadLocalStorageCreate(); + if (!state) { + mutexUnlock(&jvmMutex); + fprintf(stderr, "getJNIEnv: Unable to create ThreadLocalState\n"); + return NULL; + } + if (threadLocalStorageSet(state)) { + mutexUnlock(&jvmMutex); + goto fail; + } + THREAD_LOCAL_STORAGE_SET_QUICK(state); + + state->env = getGlobalJNIEnv(); + mutexUnlock(&jvmMutex); + if (!state->env) { + goto fail; + } + return state->env; + +fail: + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + hdfsThreadDestructor(state); + return NULL; +} + +char* getLastTLSExceptionRootCause() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionRootCause; +} + +char* getLastTLSExceptionStackTrace() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionStackTrace; +} + +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return; + } + mutexUnlock(&jvmMutex); + } + + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = (char*)rootCause; + state->lastExceptionStackTrace = (char*)stackTrace; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + "org/apache/hadoop/conf/Configuration", "set", + "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return newRuntimeError(env, "fetchEnum(%s, %s): failed to find class.", + className, valueName); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_3_2/jni_helper.h b/libhdfs/hdfs_3_2/jni_helper.h new file mode 100644 index 0000000..e63ce53 --- /dev/null +++ b/libhdfs/hdfs_3_2/jni_helper.h @@ -0,0 +1,196 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include +#include + +#include +#include +#include + +#define PATH_SEPARATOR ':' + + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * className: Name of the class where the method can be found + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, const char *className, const char *methName, + const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, const char *className, + const char *ctorSignature, ...); + +jthrowable methodIdFromClass(const char *className, const char *methName, + const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +jthrowable globalClassReference(const char *className, JNIEnv *env, jclass *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * It gets this from the ThreadLocalState if it exists. If a ThreadLocalState + * does not exist, one will be created. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Get the last exception root cause that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The root cause as a C-string. + */ +char* getLastTLSExceptionRootCause(); + +/** + * Get the last exception stack trace that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The stack trace as a C-string. + */ +char* getLastTLSExceptionStackTrace(); + +/** setTLSExceptionStrings: Sets the 'rootCause' and 'stackTrace' in the + * ThreadLocalState if one exists for the current thread. + * + * @param rootCause A string containing the root cause of an exception. + * @param stackTrace A string containing the stack trace of an exception. + * @return None. + */ +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_3_2/os/mutexes.h b/libhdfs/hdfs_3_2/os/mutexes.h new file mode 100644 index 0000000..da30bf4 --- /dev/null +++ b/libhdfs/hdfs_3_2/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting the class reference hash table. */ +extern mutex hdfsHashMutex; + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/libhdfs/hdfs_3_2/os/posix/mutexes.c b/libhdfs/hdfs_3_2/os/posix/mutexes.c new file mode 100644 index 0000000..20dafaa --- /dev/null +++ b/libhdfs/hdfs_3_2/os/posix/mutexes.c @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex hdfsHashMutex = PTHREAD_MUTEX_INITIALIZER; +mutex jvmMutex; +pthread_mutexattr_t jvmMutexAttr; + +__attribute__((constructor)) static void init() { + pthread_mutexattr_init(&jvmMutexAttr); + pthread_mutexattr_settype(&jvmMutexAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&jvmMutex, &jvmMutexAttr); +} + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/libhdfs/hdfs_3_2/os/posix/platform.h b/libhdfs/hdfs_3_2/os/posix/platform.h new file mode 100644 index 0000000..c63bbf9 --- /dev/null +++ b/libhdfs/hdfs_3_2/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/libhdfs/hdfs_3_2/os/posix/thread.c b/libhdfs/hdfs_3_2/os/posix/thread.c new file mode 100644 index 0000000..af0c61f --- /dev/null +++ b/libhdfs/hdfs_3_2/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/libhdfs/hdfs_3_2/os/posix/thread_local_storage.c b/libhdfs/hdfs_3_2/os/posix/thread_local_storage.c new file mode 100644 index 0000000..e6b59d6 --- /dev/null +++ b/libhdfs/hdfs_3_2/os/posix/thread_local_storage.c @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + struct ThreadLocalState *state = (struct ThreadLocalState*)v; + JNIEnv *env = state->env;; + jint ret; + + /* Detach the current thread from the JVM */ + if (env) { + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } + } + + /* Free exception strings */ + if (state->lastExceptionStackTrace) free(state->lastExceptionStackTrace); + if (state->lastExceptionRootCause) free(state->lastExceptionRootCause); + + /* Free the state itself */ + free(state); +} + +struct ThreadLocalState* threadLocalStorageCreate() +{ + struct ThreadLocalState *state; + state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState)); + if (state == NULL) { + fprintf(stderr, + "threadLocalStorageSet: OOM - Unable to allocate thread local state\n"); + return NULL; + } + state->lastExceptionStackTrace = NULL; + state->lastExceptionRootCause = NULL; + return state; +} + +int threadLocalStorageGet(struct ThreadLocalState **state) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *state = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(struct ThreadLocalState *state) +{ + int ret = pthread_setspecific(gTlsKey, state); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(state); + } + return ret; +} diff --git a/libhdfs/hdfs_3_2/os/thread.h b/libhdfs/hdfs_3_2/os/thread.h new file mode 100644 index 0000000..ae425d3 --- /dev/null +++ b/libhdfs/hdfs_3_2/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/libhdfs/hdfs_3_2/os/thread_local_storage.h b/libhdfs/hdfs_3_2/os/thread_local_storage.h new file mode 100644 index 0000000..025ceff --- /dev/null +++ b/libhdfs/hdfs_3_2/os/thread_local_storage.h @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) \ + static __thread struct ThreadLocalState *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + *state = quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) \ + { \ + quickTlsEnv = (state); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) +#endif + +struct ThreadLocalState { + /* The JNIEnv associated with the current thread */ + JNIEnv *env; + /* The last exception stack trace that occured on this thread */ + char *lastExceptionStackTrace; + /* The last exception root cause that occured on this thread */ + char *lastExceptionRootCause; +}; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v); + +/** + * Creates an object of ThreadLocalState. + * + * @return The newly created object if successful, NULL otherwise. + */ +struct ThreadLocalState* threadLocalStorageCreate(); + +/** + * Gets the ThreadLocalState in thread-local storage for the current thread. + * If the call succeeds, and there is a ThreadLocalState associated with this + * thread, then returns 0 and populates 'state'. If the call succeeds, but + * there is no ThreadLocalState associated with this thread, then returns 0 + * and sets ThreadLocalState to NULL. If the call fails, then returns non-zero. + * Only one thread at a time may execute this function. The caller is + * responsible for enforcing mutual exclusion. + * + * @param env ThreadLocalState out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(struct ThreadLocalState **state); + +/** + * Sets the ThreadLocalState in thread-local storage for the current thread. + * + * @param env ThreadLocalState to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(struct ThreadLocalState *state); + +#endif diff --git a/libhdfs/hdfs_3_2/os/windows/inttypes.h b/libhdfs/hdfs_3_2/os/windows/inttypes.h new file mode 100644 index 0000000..a520d15 --- /dev/null +++ b/libhdfs/hdfs_3_2/os/windows/inttypes.h @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_INTTYPES_H +#define LIBHDFS_INTTYPES_H + +/* On Windows, inttypes.h does not exist, so manually define what we need. */ + +#define PRId64 "I64d" +#define PRIu64 "I64u" +typedef unsigned __int64 uint64_t; + +#endif diff --git a/libhdfs/hdfs_3_2/os/windows/mutexes.c b/libhdfs/hdfs_3_2/os/windows/mutexes.c new file mode 100644 index 0000000..875f033 --- /dev/null +++ b/libhdfs/hdfs_3_2/os/windows/mutexes.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include + +mutex hdfsHashMutex; +mutex jvmMutex; + +/** + * Unfortunately, there is no simple static initializer for a critical section. + * Instead, the API requires calling InitializeCriticalSection. Since libhdfs + * lacks an explicit initialization function, there is no obvious existing place + * for the InitializeCriticalSection calls. To work around this, we define an + * initialization function and instruct the linker to set a pointer to that + * function as a user-defined global initializer. See discussion of CRT + * Initialization: + * http://msdn.microsoft.com/en-us/library/bb918180.aspx + */ +static void __cdecl initializeMutexes(void) { + InitializeCriticalSection(&hdfsHashMutex); + InitializeCriticalSection(&jvmMutex); +} +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) +const void (__cdecl *pInitialize)(void) = initializeMutexes; + +int mutexLock(mutex *m) { + EnterCriticalSection(m); + return 0; +} + +int mutexUnlock(mutex *m) { + LeaveCriticalSection(m); + return 0; +} diff --git a/libhdfs/hdfs_3_2/os/windows/platform.h b/libhdfs/hdfs_3_2/os/windows/platform.h new file mode 100644 index 0000000..9eedfde --- /dev/null +++ b/libhdfs/hdfs_3_2/os/windows/platform.h @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include +#include +#include + +/* + * O_ACCMODE defined to match Linux definition. + */ +#ifndef O_ACCMODE +#define O_ACCMODE 0x0003 +#endif + +/* + * Windows has a different name for its maximum path length constant. + */ +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +/* + * Windows does not define EDQUOT and ESTALE in errno.h. The closest equivalents + * are these constants from winsock.h. + */ +#ifndef EDQUOT +#define EDQUOT WSAEDQUOT +#endif + +#ifndef ESTALE +#define ESTALE WSAESTALE +#endif + +/* + * gcc-style type-checked format arguments are not supported on Windows, so just + * stub this macro. + */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) + +/* + * Define macros for various string formatting functions not defined on Windows. + * Where possible, we reroute to one of the secure CRT variants. On Windows, + * the preprocessor does support variadic macros, even though they weren't + * defined until C99. + */ +#define snprintf(str, size, format, ...) \ + _snprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) +#define strncpy(dest, src, n) \ + strncpy_s((dest), (n), (src), _TRUNCATE) +#define strtok_r(str, delim, saveptr) \ + strtok_s((str), (delim), (saveptr)) +#define vsnprintf(str, size, format, ...) \ + vsnprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) + +/* + * Mutex data type defined as Windows CRITICAL_SECTION. A critical section (not + * Windows mutex) is used, because libhdfs only needs synchronization of multiple + * threads within a single process, not synchronization across process + * boundaries. + */ +typedef CRITICAL_SECTION mutex; + +/* + * Thread data type defined as HANDLE to a Windows thread. + */ +typedef HANDLE threadId; + +#endif diff --git a/libhdfs/hdfs_3_2/os/windows/thread.c b/libhdfs/hdfs_3_2/os/windows/thread.c new file mode 100644 index 0000000..f5cc2a7 --- /dev/null +++ b/libhdfs/hdfs_3_2/os/windows/thread.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by CreateThread. + * + * @param toRun thread to run + * @return DWORD result of running thread (always 0) + */ +static DWORD WINAPI runThread(LPVOID toRun) { + const thread *t = toRun; + t->start(t->arg); + return 0; +} + +int threadCreate(thread *t) { + DWORD ret = 0; + HANDLE h; + h = CreateThread(NULL, 0, runThread, t, 0, NULL); + if (h) { + t->id = h; + } else { + ret = GetLastError(); + fprintf(stderr, "threadCreate: CreateThread failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + DWORD ret = WaitForSingleObject(t->id, INFINITE); + switch (ret) { + case WAIT_OBJECT_0: + break; + case WAIT_FAILED: + ret = GetLastError(); + fprintf(stderr, "threadJoin: WaitForSingleObject failed with error %d\n", + ret); + break; + default: + fprintf(stderr, "threadJoin: WaitForSingleObject unexpected error %d\n", + ret); + break; + } + return ret; +} diff --git a/libhdfs/hdfs_3_2/os/windows/thread_local_storage.c b/libhdfs/hdfs_3_2/os/windows/thread_local_storage.c new file mode 100644 index 0000000..28d014d --- /dev/null +++ b/libhdfs/hdfs_3_2/os/windows/thread_local_storage.c @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +/** Key that allows us to retrieve thread-local storage */ +static DWORD gTlsIndex = TLS_OUT_OF_INDEXES; + +/** + * If the current thread has a JNIEnv in thread-local storage, then detaches the + * current thread from the JVM and also frees up the ThreadLocalState object. + */ +static void detachCurrentThreadFromJvm() +{ + struct ThreadLocalState *state = NULL; + JNIEnv *env = NULL; + JavaVM *vm; + jint ret; + if (threadLocalStorageGet(&state) || !state) { + return; + } + if (!state->env) { + return; + } + env = state->env; + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, + "detachCurrentThreadFromJvm: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + (*vm)->DetachCurrentThread(vm); + } + + /* Free exception strings */ + if (state->lastExceptionStackTrace) free(state->lastExceptionStackTrace); + if (state->lastExceptionRootCause) free(state->lastExceptionRootCause); + + /* Free the state itself */ + free(state); +} + +void hdfsThreadDestructor(void *v) +{ + // Ignore 'v' since it will contain the state and we will obtain it in the below + // call anyway. + detachCurrentThreadFromJvm(); +} + +/** + * Unlike pthreads, the Windows API does not seem to provide a convenient way to + * hook a callback onto thread shutdown. However, the Windows portable + * executable format does define a concept of thread-local storage callbacks. + * Here, we define a function and instruct the linker to set a pointer to that + * function in the segment for thread-local storage callbacks. See page 85 of + * Microsoft Portable Executable and Common Object File Format Specification: + * http://msdn.microsoft.com/en-us/gg463119.aspx + * This technique only works for implicit linking (OS loads DLL on demand), not + * for explicit linking (user code calls LoadLibrary directly). This effectively + * means that we have a known limitation: libhdfs may not work correctly if a + * Windows application attempts to use it via explicit linking. + * + * @param h module handle + * @param reason the reason for calling the callback + * @param pv reserved, unused + */ +static void NTAPI tlsCallback(PVOID h, DWORD reason, PVOID pv) +{ + DWORD tlsIndex; + switch (reason) { + case DLL_THREAD_DETACH: + detachCurrentThreadFromJvm(); + break; + case DLL_PROCESS_DETACH: + detachCurrentThreadFromJvm(); + tlsIndex = gTlsIndex; + gTlsIndex = TLS_OUT_OF_INDEXES; + if (!TlsFree(tlsIndex)) { + fprintf(stderr, "tlsCallback: TlsFree failed with error %d\n", + GetLastError()); + } + break; + default: + break; + } +} + +/* + * A variable named _tls_used contains the TLS directory, which contains a list + * of pointers to callback functions. Normally, the linker won't retain this + * variable unless the executable has implicit thread-local variables, defined + * using the __declspec(thread) extended storage-class modifier. libhdfs + * doesn't use __declspec(thread), and we have no guarantee that the executable + * linked to libhdfs will use __declspec(thread). By forcing the linker to + * reference _tls_used, we guarantee that the binary retains the TLS directory. + * See Microsoft Visual Studio 10.0/VC/crt/src/tlssup.c . + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#endif + +/* + * We must retain a pointer to the callback function. Force the linker to keep + * this symbol, even though it appears that nothing in our source code uses it. + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:pTlsCallback") +#else +#pragma comment(linker, "/INCLUDE:_pTlsCallback") +#endif + +/* + * Define constant pointer to our callback, and tell the linker to pin it into + * the TLS directory so that it receives thread callbacks. Use external linkage + * to protect against the linker discarding the seemingly unused symbol. + */ +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK pTlsCallback; +const PIMAGE_TLS_CALLBACK pTlsCallback = tlsCallback; +#pragma const_seg() + +struct ThreadLocalState* threadLocalStorageCreate() +{ + struct ThreadLocalState *state; + state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState)); + if (state == NULL) { + fprintf(stderr, + "threadLocalStorageSet: OOM - Unable to allocate thread local state\n"); + return NULL; + } + state->lastExceptionStackTrace = NULL; + state->lastExceptionRootCause = NULL; + return state; +} + +int threadLocalStorageGet(struct ThreadLocalState **state) +{ + LPVOID tls; + DWORD ret; + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + gTlsIndex = TlsAlloc(); + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + fprintf(stderr, + "threadLocalStorageGet: TlsAlloc failed with error %d\n", + TLS_OUT_OF_INDEXES); + return TLS_OUT_OF_INDEXES; + } + } + tls = TlsGetValue(gTlsIndex); + if (tls) { + *state = tls; + return 0; + } else { + ret = GetLastError(); + if (ERROR_SUCCESS == ret) { + /* Thread-local storage contains NULL, because we haven't set it yet. */ + *state = NULL; + return 0; + } else { + /* + * The API call failed. According to documentation, TlsGetValue cannot + * fail as long as the index is a valid index from a successful TlsAlloc + * call. This error handling is purely defensive. + */ + fprintf(stderr, + "threadLocalStorageGet: TlsGetValue failed with error %d\n", ret); + return ret; + } + } +} + +int threadLocalStorageSet(struct ThreadLocalState *state) +{ + DWORD ret = 0; + if (!TlsSetValue(gTlsIndex, (LPVOID)state)) { + ret = GetLastError(); + fprintf(stderr, + "threadLocalStorageSet: TlsSetValue failed with error %d\n", + ret); + detachCurrentThreadFromJvm(state); + } + return ret; +} diff --git a/libhdfs/hdfs_3_2/os/windows/unistd.h b/libhdfs/hdfs_3_2/os/windows/unistd.h new file mode 100644 index 0000000..b82ce48 --- /dev/null +++ b/libhdfs/hdfs_3_2/os/windows/unistd.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_UNISTD_H +#define LIBHDFS_UNISTD_H + +/* On Windows, unistd.h does not exist, so manually define what we need. */ + +#include /* Declares getpid(). */ +#include + +/* Re-route sleep to Sleep, converting units from seconds to milliseconds. */ +#define sleep(seconds) Sleep((seconds) * 1000) +#endif diff --git a/libhdfs/hdfs_3_3/exception.c b/libhdfs/hdfs_3_3/exception.c new file mode 100644 index 0000000..fec9a10 --- /dev/null +++ b/libhdfs/hdfs_3_3/exception.c @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "java.lang.UnsupportedOperationException", + 0, + ENOTSUP, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +/** + * getExceptionUtilString: A helper function that calls 'methodName' in + * ExceptionUtils. The function 'methodName' should have a return type of a + * java String. + * + * @param env The JNI environment. + * @param exc The exception to get information for. + * @param methodName The method of ExceptionUtils to call that has a String + * return type. + * + * @return A C-type string containing the string returned by + * ExceptionUtils.'methodName', or NULL on failure. + */ +static char* getExceptionUtilString(JNIEnv *env, jthrowable exc, char *methodName) +{ + jthrowable jthr; + jvalue jVal; + jstring jStr = NULL; + char *excString = NULL; + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_EXCEPTION_UTILS, + methodName, "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + jStr = jVal.l; + jthr = newCStr(env, jStr, &excString); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + destroyLocalReference(env, jStr); + return excString; +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jthrowable jthr; + const char *stackTrace; + const char *rootCause; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + rootCause = getExceptionUtilString(env, exc, "getRootCauseMessage"); + stackTrace = getExceptionUtilString(env, exc, "getStackTrace"); + // Save the exception details in the thread-local state. + setTLSExceptionStrings(rootCause, stackTrace); + + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + if (!rootCause) { + fprintf(stderr, "(unable to get root cause for %s)\n", className); + } else { + fprintf(stderr, "%s", rootCause); + } + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + } + } + + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/libhdfs/hdfs_3_3/exception.h b/libhdfs/hdfs_3_3/exception.h new file mode 100644 index 0000000..cdf93a1 --- /dev/null +++ b/libhdfs/hdfs_3_3/exception.h @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * printExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + * + * The root cause and stack trace exception strings retrieved from the last + * exception that happened on a thread are stored in the corresponding + * thread local state and are accessed by hdfsGetLastExceptionRootCause and + * hdfsGetLastExceptionStackTrace respectively. + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Store the information about the pending exception in the thread-local state + * and print it and free the jthrowable object. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/libhdfs/hdfs_3_3/hdfs.c b/libhdfs/hdfs_3_3/hdfs.c new file mode 100644 index 0000000..60f2826 --- /dev/null +++ b/libhdfs/hdfs_3_3/hdfs.c @@ -0,0 +1,3831 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) +#define HDFS_FILE_SUPPORTS_DIRECT_PREAD (1<<1) + +/** + * Reads bytes using the read(ByteBuffer) API. By using Java + * DirectByteBuffers we can avoid copying the bytes onto the Java heap. + * Instead the data will be directly copied from kernel space to the C heap. + */ +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); + +/** + * Reads bytes using the read(long, ByteBuffer) API. By using Java + * DirectByteBuffers we can avoid copying the bytes onto the Java heap. + * Instead the data will be directly copied from kernel space to the C heap. + */ +tSize preadDirect(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, + tSize length); + +int preadFullyDirect(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, + tSize length); + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics) +{ + jthrowable jthr; + jobject hedgedReadMetrics = NULL; + jvalue jVal; + struct hdfsHedgedReadMetrics *m = NULL; + int ret; + jobject jFS = (jobject)fs; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + JC_DISTRIBUTED_FILE_SYSTEM, "getHedgedReadMetrics", + "()Lorg/apache/hadoop/hdfs/DFSHedgedReadMetrics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadMetrics: getHedgedReadMetrics failed"); + goto done; + } + hedgedReadMetrics = jVal.l; + + m = malloc(sizeof(struct hdfsHedgedReadMetrics)); + if (!m) { + ret = ENOMEM; + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + JC_DFS_HEDGED_READ_METRICS, "getHedgedReadOps", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOps failed"); + goto done; + } + m->hedgedReadOps = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + JC_DFS_HEDGED_READ_METRICS, "getHedgedReadWins", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadWins failed"); + goto done; + } + m->hedgedReadOpsWin = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + JC_DFS_HEDGED_READ_METRICS, "getHedgedReadOpsInCurThread", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOpsInCurThread failed"); + goto done; + } + m->hedgedReadOpsInCurThread = jVal.j; + + *metrics = m; + m = NULL; + ret = 0; + +done: + destroyLocalReference(env, hedgedReadMetrics); + free(m); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics) +{ + free(metrics); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + JC_HDFS_DATA_INPUT_STREAM, "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalShortCircuitBytesRead", + "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalZeroCopyBytesRead", + "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +int hdfsFileClearReadStatistics(hdfsFile file) +{ + jthrowable jthr; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return EINTERNAL; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + JC_HDFS_DATA_INPUT_STREAM, "clearReadStatistics", + "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileClearReadStatistics: clearReadStatistics failed"); + goto done; + } + ret = 0; +done: + if (ret) { + errno = ret; + return ret; + } + return 0; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return (file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) != 0; +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsFileUsesDirectPread(hdfsFile file) +{ + return (file->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD) != 0; +} + +void hdfsFileDisableDirectPread(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_PREAD; +} + + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, JC_DOMAIN_SOCKET, + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfCachedClass(env, &jPath, JC_PATH, + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "getInt", + JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "newInstanceLocal", + JMETHOD1(JPARAM(HADOOP_CONF), JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_URI, "create", + "(Ljava/lang/String;)Ljava/net/URI;", jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "newInstance", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI, + jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI, + jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), + "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + struct hdfsStreamBuilder *bld = hdfsStreamBuilderAlloc(fs, path, flags); + if (bufferSize != 0) { + hdfsStreamBuilderSetBufferSize(bld, bufferSize); + } + if (replication != 0) { + hdfsStreamBuilderSetReplication(bld, replication); + } + if (blockSize != 0) { + hdfsStreamBuilderSetDefaultBlockSize(bld, blockSize); + } + return hdfsStreamBuilderBuild(bld); +} + +struct hdfsStreamBuilder { + hdfsFS fs; + int flags; + int32_t bufferSize; + int16_t replication; + int64_t defaultBlockSize; + char path[1]; +}; + +struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs, + const char *path, int flags) +{ + size_t path_len = strlen(path); + struct hdfsStreamBuilder *bld; + + // Check for overflow in path_len + if (path_len > SIZE_MAX - sizeof(struct hdfsStreamBuilder)) { + errno = EOVERFLOW; + return NULL; + } + // sizeof(hdfsStreamBuilder->path) includes one byte for the string + // terminator + bld = malloc(sizeof(struct hdfsStreamBuilder) + path_len); + if (!bld) { + errno = ENOMEM; + return NULL; + } + bld->fs = fs; + bld->flags = flags; + bld->bufferSize = 0; + bld->replication = 0; + bld->defaultBlockSize = 0; + memcpy(bld->path, path, path_len); + bld->path[path_len] = '\0'; + return bld; +} + +void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld) +{ + free(bld); +} + +int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld, + int32_t bufferSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->bufferSize = bufferSize; + return 0; +} + +int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld, + int16_t replication) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->replication = replication; + return 0; +} + +int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld, + int64_t defaultBlockSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->defaultBlockSize = defaultBlockSize; + return 0; +} + +/** + * Delegates to FsDataInputStream#hasCapability(String). Used to check if a + * given input stream supports certain methods, such as + * ByteBufferReadable#read(ByteBuffer). + * + * @param jFile the FsDataInputStream to call hasCapability on + * @param capability the name of the capability to query; for a full list of + * possible values see StreamCapabilities + * + * @return true if the given jFile has the given capability, false otherwise + * + * @see org.apache.hadoop.fs.StreamCapabilities + */ +static int hdfsHasStreamCapability(jobject jFile, + const char *capability) { + int ret = 0; + jthrowable jthr = NULL; + jvalue jVal; + jstring jCapabilityString = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return 0; + } + + jthr = newJavaStr(env, capability, &jCapabilityString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHasStreamCapability(%s): newJavaStr", capability); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFile, + JC_FS_DATA_INPUT_STREAM, "hasCapability", "(Ljava/lang/String;)Z", + jCapabilityString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHasStreamCapability(%s): FSDataInputStream#hasCapability", + capability); + goto done; + } + +done: + destroyLocalReference(env, jthr); + destroyLocalReference(env, jCapabilityString); + if (ret) { + errno = ret; + return 0; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, + int32_t bufferSize, int16_t replication, int64_t blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", + accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, + "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_FSDISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSDOSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_FSDOSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "getInt", + "(Ljava/lang/String;I)I", jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "getInt", + "(Ljava/lang/String;I)I", jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + method, signature, jPath, jOverWrite, jBufferSize, + jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Check the StreamCapabilities of jFile to see if we can do direct + // reads + if (hdfsHasStreamCapability(jFile, "in:readbytebuffer")) { + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } + + // Check the StreamCapabilities of jFile to see if we can do direct + // preads + if (hdfsHasStreamCapability(jFile, "in:preadbytebuffer")) { + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_PREAD; + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld) +{ + hdfsFile file = hdfsOpenFileImpl(bld->fs, bld->path, bld->flags, + bld->bufferSize, bld->replication, bld->defaultBlockSize); + int prevErrno = errno; + hdfsStreamBuilderFree(bld); + errno = prevErrno; + return file; +} + +int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) +{ + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + jobject jPath = NULL; + + JNIEnv *env = getJNIEnv(); + + if (!env) { + errno = EINTERNAL; + return -1; + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): constructNewObjectOfPath", path); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "truncate", JMETHOD2(JPARAM(HADOOP_PATH), "J", "Z"), + jPath, newlength); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): FileSystem#truncate", path); + return -1; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +int hdfsUnbufferFile(hdfsFile file) +{ + int ret; + jthrowable jthr; + JNIEnv *env = getJNIEnv(); + + if (!env) { + ret = EINTERNAL; + goto done; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = ENOTSUP; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + JC_FS_DATA_INPUT_STREAM, "unbuffer", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + HADOOP_FSDISTRM "#unbuffer failed:"); + goto done; + } + ret = 0; + +done: + errno = ret; + return ret; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + CachedJavaClass cachedJavaClass; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + if (file->type == HDFS_STREAM_INPUT) { + cachedJavaClass = JC_FS_DATA_INPUT_STREAM; + } else { + cachedJavaClass = JC_FS_DATA_OUTPUT_STREAM; + } + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + cachedJavaClass, "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +/** + * If the underlying stream supports the ByteBufferReadable interface then + * this method will transparently use read(ByteBuffer). This can help + * improve performance as it avoids unnecessarily copying data on to the Java + * heap. Instead the data will be directly copied from kernel space to the C + * heap. + */ +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + // We only copy the portion of the jbRarray that was actually filled by + // the call to FsDataInputStream#read; #read is not guaranteed to fill the + // entire buffer, instead it returns the number of bytes read into the + // buffer; we use the return value as the input in GetByteArrayRegion to + // ensure don't copy more bytes than necessary + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer buf = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(buf); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "read", + "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + // Reached EOF, return 0 + if (jVal.i < 0) { + return 0; + } + // 0 bytes read, return error + if (jVal.i == 0) { + errno = EINTR; + return -1; + } + return jVal.i; +} + +/** + * If the underlying stream supports the ByteBufferPositionedReadable + * interface then this method will transparently use read(long, ByteBuffer). + * This can help improve performance as it avoids unnecessarily copying data + * on to the Java heap. Instead the data will be directly copied from kernel + * space to the C heap. + */ +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD) { + return preadDirect(fs, f, position, buffer, length); + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "read", "(J[BII)I", position, + jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize preadDirect(hdfsFS fs, hdfsFile f, tOffset position, void* buffer, + tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer buf = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(position, buf); + + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "read", "(JLjava/nio/ByteBuffer;)I", + position, bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "preadDirect: FSDataInputStream#read"); + return -1; + } + // Reached EOF, return 0 + if (jVal.i < 0) { + return 0; + } + // 0 bytes read, return error + if (jVal.i == 0) { + errno = EINTR; + return -1; + } + return jVal.i; +} + +/** + * Like hdfsPread, if the underlying stream supports the + * ByteBufferPositionedReadable interface then this method will transparently + * use readFully(long, ByteBuffer). + */ +int hdfsPreadFully(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) { + JNIEnv* env; + jbyteArray jbRarray; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD) { + return preadFullyDirect(fs, f, position, buffer, length); + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "readFully", "(J[BII)V", + position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + + (*env)->GetByteArrayRegion(env, jbRarray, 0, length, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return 0; +} + +int preadFullyDirect(hdfsFS fs, hdfsFile f, tOffset position, void* buffer, + tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer buf = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(position, buf); + + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "readFully", + "(JLjava/nio/ByteBuffer;)V", position, bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "preadDirect: FSDataInputStream#read"); + return -1; + } + return 0; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + JC_FS_DATA_OUTPUT_STREAM, "write", "([B)V", + jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + CachedJavaClass cachedJavaClass; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + if (f->type == HDFS_STREAM_INPUT) { + cachedJavaClass = JC_FS_DATA_INPUT_STREAM; + } else { + cachedJavaClass = JC_FS_DATA_OUTPUT_STREAM; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + cachedJavaClass, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + JC_FS_DATA_OUTPUT_STREAM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + JC_FS_DATA_OUTPUT_STREAM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + JC_FS_DATA_OUTPUT_STREAM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_UTIL, + "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", jPath, + jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "rename", JMETHOD2(JPARAM(HADOOP_PATH), JPARAM + (HADOOP_PATH), "Z"), jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getWorkingDirectory", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, JC_PATH, "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setWorkingDirectory", "(Lorg/apache/hadoop/fs/Path;)V", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfCachedClass(env, &jPermObj, JC_FS_PERMISSION, + "(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfCachedClass(%s)", HADOOP_FSPERM); + goto done; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setPermission", JMETHOD2(JPARAM(HADOOP_PATH), + JPARAM(HADOOP_FSPERM), JAVA_VOID), jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", + JAVA_VOID), jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + jobject globalByteBufferPool = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + ret = EINVAL; + goto done; + } + // Only set opts->byteBufferPool if creating a global reference is + // successful + globalByteBufferPool = (*env)->NewGlobalRef(env, byteBufferPool); + if (!globalByteBufferPool) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", + className); + ret = EINVAL; + goto done; + } + // Delete any previous ByteBufferPool we had before setting a new one. + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = globalByteBufferPool; + } else if (opts->byteBufferPool) { + // If the specified className is NULL, delete any previous + // ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + ret = 0; +done: + destroyLocalReference(env, byteBufferPool); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, HADOOP_RO, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_ENUM_SET, + "of", "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, HADOOP_RO); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_ENUM_SET, + "noneOf", "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + JC_BYTE_BUFFER, "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + JC_BYTE_BUFFER, "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + JC_BYTE_BUFFER, "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + destroyLocalReference(env, exc); + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + JC_FS_DATA_INPUT_STREAM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + JC_FS_DATA_INPUT_STREAM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", jFileStatus, start, + length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + jthr = (*env)->ExceptionOccurred(env); + if (jthr || !jFileBlock) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, + JC_BLOCK_LOCATION, "getHosts", + "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + jthr = (*env)->ExceptionOccurred(env); + if (jthr || !jHost) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + errno = ret; + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, (jlong *)&blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, + JC_FS_STATUS, "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, JC_FS_STATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, "isDir", + "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, JC_PATH, "toString", + "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, "getOwner", + "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_FILESTAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, + JC_FS_PERMISSION, "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, "exists", + JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getFileStatus", JMETHOD1(JPARAM(HADOOP_PATH), JPARAM + (HADOOP_FILESTAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + JC_DISTRIBUTED_FILE_SYSTEM, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_FILESTAT)), jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + jthr = (*env)->ExceptionOccurred(env); + if (jthr || !tmpStat) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + errno = 0; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + +char* hdfsGetLastExceptionRootCause() +{ + return getLastTLSExceptionRootCause(); +} + +char* hdfsGetLastExceptionStackTrace() +{ + return getLastTLSExceptionStackTrace(); +} + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/docs/include/hdfs_abi_3_3.h b/libhdfs/hdfs_3_3/include/hdfs/hdfs.h similarity index 100% rename from docs/include/hdfs_abi_3_3.h rename to libhdfs/hdfs_3_3/include/hdfs/hdfs.h diff --git a/libhdfs/hdfs_3_3/jclasses.c b/libhdfs/hdfs_3_3/jclasses.c new file mode 100644 index 0000000..cf880e9 --- /dev/null +++ b/libhdfs/hdfs_3_3/jclasses.c @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "os/mutexes.h" + +#include + +/** + * Whether initCachedClasses has been called or not. Protected by the mutex + * jclassInitMutex. + */ +static int jclassesInitialized = 0; + +typedef struct { + jclass javaClass; + const char *className; +} javaClassAndName; + +/** + * A collection of commonly used jclass objects that are used throughout + * libhdfs. The jclasses are loaded immediately after the JVM is created (see + * initCachedClasses). The array is indexed using CachedJavaClass. + */ +javaClassAndName cachedJavaClasses[NUM_CACHED_CLASSES]; + +/** + * Helper method that creates and sets a jclass object given a class name. + * Returns a jthrowable on error, NULL otherwise. + */ +static jthrowable initCachedClass(JNIEnv *env, const char *className, + jclass *cachedJclass) { + assert(className != NULL && "Found a CachedJavaClass without a class " + "name"); + jthrowable jthr = NULL; + jclass tempLocalClassRef; + tempLocalClassRef = (*env)->FindClass(env, className); + if (!tempLocalClassRef) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *cachedJclass = (jclass) (*env)->NewGlobalRef(env, tempLocalClassRef); + if (!*cachedJclass) { + jthr = getPendingExceptionAndClear(env); + goto done; + } +done: + destroyLocalReference(env, tempLocalClassRef); + return jthr; +} + +jthrowable initCachedClasses(JNIEnv* env) { + mutexLock(&jclassInitMutex); + if (!jclassesInitialized) { + // Set all the class names + cachedJavaClasses[JC_CONFIGURATION].className = + "org/apache/hadoop/conf/Configuration"; + cachedJavaClasses[JC_PATH].className = + "org/apache/hadoop/fs/Path"; + cachedJavaClasses[JC_FILE_SYSTEM].className = + "org/apache/hadoop/fs/FileSystem"; + cachedJavaClasses[JC_FS_STATUS].className = + "org/apache/hadoop/fs/FsStatus"; + cachedJavaClasses[JC_FILE_UTIL].className = + "org/apache/hadoop/fs/FileUtil"; + cachedJavaClasses[JC_BLOCK_LOCATION].className = + "org/apache/hadoop/fs/BlockLocation"; + cachedJavaClasses[JC_DFS_HEDGED_READ_METRICS].className = + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics"; + cachedJavaClasses[JC_DISTRIBUTED_FILE_SYSTEM].className = + "org/apache/hadoop/hdfs/DistributedFileSystem"; + cachedJavaClasses[JC_FS_DATA_INPUT_STREAM].className = + "org/apache/hadoop/fs/FSDataInputStream"; + cachedJavaClasses[JC_FS_DATA_OUTPUT_STREAM].className = + "org/apache/hadoop/fs/FSDataOutputStream"; + cachedJavaClasses[JC_FILE_STATUS].className = + "org/apache/hadoop/fs/FileStatus"; + cachedJavaClasses[JC_FS_PERMISSION].className = + "org/apache/hadoop/fs/permission/FsPermission"; + cachedJavaClasses[JC_READ_STATISTICS].className = + "org/apache/hadoop/hdfs/ReadStatistics"; + cachedJavaClasses[JC_HDFS_DATA_INPUT_STREAM].className = + "org/apache/hadoop/hdfs/client/HdfsDataInputStream"; + cachedJavaClasses[JC_DOMAIN_SOCKET].className = + "org/apache/hadoop/net/unix/DomainSocket"; + cachedJavaClasses[JC_URI].className = + "java/net/URI"; + cachedJavaClasses[JC_BYTE_BUFFER].className = + "java/nio/ByteBuffer"; + cachedJavaClasses[JC_ENUM_SET].className = + "java/util/EnumSet"; + cachedJavaClasses[JC_EXCEPTION_UTILS].className = + "org/apache/commons/lang3/exception/ExceptionUtils"; + + // Create and set the jclass objects based on the class names set above + jthrowable jthr; + int numCachedClasses = + sizeof(cachedJavaClasses) / sizeof(javaClassAndName); + for (int i = 0; i < numCachedClasses; i++) { + jthr = initCachedClass(env, cachedJavaClasses[i].className, + &cachedJavaClasses[i].javaClass); + if (jthr) { + mutexUnlock(&jclassInitMutex); + return jthr; + } + } + jclassesInitialized = 1; + } + mutexUnlock(&jclassInitMutex); + return NULL; +} + +jclass getJclass(CachedJavaClass cachedJavaClass) { + return cachedJavaClasses[cachedJavaClass].javaClass; +} + +const char *getClassName(CachedJavaClass cachedJavaClass) { + return cachedJavaClasses[cachedJavaClass].className; +} diff --git a/libhdfs/hdfs_3_3/jclasses.h b/libhdfs/hdfs_3_3/jclasses.h new file mode 100644 index 0000000..92cdd54 --- /dev/null +++ b/libhdfs/hdfs_3_3/jclasses.h @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JCLASSES_H +#define LIBHDFS_JCLASSES_H + +#include + +/** + * Encapsulates logic to cache jclass objects so they can re-used across + * calls to FindClass. Creating jclass objects every time libhdfs has to + * invoke a method can hurt performance. By cacheing jclass objects we avoid + * this overhead. + * + * We use the term "cached" here loosely; jclasses are not truly cached, + * instead they are created once during JVM load and are kept alive until the + * process shutdowns. There is no eviction of jclass objects. + * + * @see https://www.ibm.com/developerworks/library/j-jni/index.html#notc + */ + +/** + * Each enum value represents one jclass that is cached. Enum values should + * be passed to getJclass or getName to get the jclass object or class name + * represented by the enum value. + */ +typedef enum { + JC_CONFIGURATION, + JC_PATH, + JC_FILE_SYSTEM, + JC_FS_STATUS, + JC_FILE_UTIL, + JC_BLOCK_LOCATION, + JC_DFS_HEDGED_READ_METRICS, + JC_DISTRIBUTED_FILE_SYSTEM, + JC_FS_DATA_INPUT_STREAM, + JC_FS_DATA_OUTPUT_STREAM, + JC_FILE_STATUS, + JC_FS_PERMISSION, + JC_READ_STATISTICS, + JC_HDFS_DATA_INPUT_STREAM, + JC_DOMAIN_SOCKET, + JC_URI, + JC_BYTE_BUFFER, + JC_ENUM_SET, + JC_EXCEPTION_UTILS, + // A special marker enum that counts the number of cached jclasses + NUM_CACHED_CLASSES +} CachedJavaClass; + +/** + * Internally initializes all jclass objects listed in the CachedJavaClass + * enum. This method is idempotent and thread-safe. + */ +jthrowable initCachedClasses(JNIEnv* env); + +/** + * Return the jclass object represented by the given CachedJavaClass + */ +jclass getJclass(CachedJavaClass cachedJavaClass); + +/** + * Return the class name represented by the given CachedJavaClass + */ +const char *getClassName(CachedJavaClass cachedJavaClass); + +/* Some frequently used HDFS class names */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_FILEUTIL "org/apache/hadoop/fs/FileUtil" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS_HRM "org/apache/hadoop/hdfs/DFSHedgedReadMetrics" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_FSDISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_FSDOSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_FILESTAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define HADOOP_RSTAT "org/apache/hadoop/hdfs/ReadStatistics" +#define HADOOP_HDISTRM "org/apache/hadoop/hdfs/client/HdfsDataInputStream" +#define HADOOP_RO "org/apache/hadoop/fs/ReadOption" +#define HADOOP_DS "org/apache/hadoop/net/unix/DomainSocket" + +/* Some frequently used Java class names */ +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_BYTEBUFFER "java/nio/ByteBuffer" +#define JAVA_STRING "java/lang/String" +#define JAVA_ENUMSET "java/util/EnumSet" + +/* Some frequently used third-party class names */ + +#define EXCEPTION_UTILS "org/apache/commons/lang3/exception/ExceptionUtils" + +#endif /*LIBHDFS_JCLASSES_H*/ diff --git a/libhdfs/hdfs_3_3/jni_helper.c b/libhdfs/hdfs_3_3/jni_helper.c new file mode 100644 index 0000000..4efb3b6 --- /dev/null +++ b/libhdfs/hdfs_3_3/jni_helper.c @@ -0,0 +1,964 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "config.h" +#include "exception.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "platform.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +/** + * Does the work to actually execute a Java method. Takes in an existing jclass + * object and a va_list of arguments for the Java method to be invoked. + */ +static jthrowable invokeMethodOnJclass(JNIEnv *env, jvalue *retval, + MethType methType, jobject instObj, jclass cls, const char *className, + const char *methName, const char *methSignature, va_list args) +{ + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = methodIdFromClass(cls, className, methName, methSignature, methType, + env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable findClassAndInvokeMethod(JNIEnv *env, jvalue *retval, + MethType methType, jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + jclass cls = NULL; + jthrowable jthr = NULL; + + va_list args; + va_start(args, methSignature); + + jthr = validateMethodType(env, methType); + if (jthr) { + goto done; + } + + cls = (*env)->FindClass(env, className); + if (!cls) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + + jthr = invokeMethodOnJclass(env, retval, methType, instObj, cls, + className, methName, methSignature, args); + +done: + va_end(args); + destroyLocalReference(env, cls); + return jthr; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, CachedJavaClass class, + const char *methName, const char *methSignature, ...) +{ + jthrowable jthr; + + va_list args; + va_start(args, methSignature); + + jthr = invokeMethodOnJclass(env, retval, methType, instObj, + getJclass(class), getClassName(class), methName, methSignature, + args); + + va_end(args); + return jthr; +} + +static jthrowable constructNewObjectOfJclass(JNIEnv *env, + jobject *out, jclass cls, const char *className, + const char *ctorSignature, va_list args) { + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = methodIdFromClass(cls, className, "", ctorSignature, INSTANCE, + env, &mid); + if (jthr) + return jthr; + jobj = (*env)->NewObjectV(env, cls, mid, args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, + const char *className, const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jthrowable jthr = NULL; + + cls = (*env)->FindClass(env, className); + if (!cls) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + + va_start(args, ctorSignature); + jthr = constructNewObjectOfJclass(env, out, cls, className, + ctorSignature, args); + va_end(args); +done: + destroyLocalReference(env, cls); + return jthr; +} + +jthrowable constructNewObjectOfCachedClass(JNIEnv *env, jobject *out, + CachedJavaClass cachedJavaClass, const char *ctorSignature, ...) +{ + jthrowable jthr = NULL; + va_list args; + va_start(args, ctorSignature); + + jthr = constructNewObjectOfJclass(env, out, + getJclass(cachedJavaClass), getClassName(cachedJavaClass), + ctorSignature, args); + + va_end(args); + return jthr; +} + +jthrowable methodIdFromClass(jclass cls, const char *className, + const char *methName, const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jthrowable jthr; + jmethodID mid = 0; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + goto done; + } + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + +/** + * For the given path, expand it by filling in with all *.jar or *.JAR files, + * separated by PATH_SEPARATOR. Assumes that expanded is big enough to hold the + * string, eg allocated after using this function with expanded=NULL to get the + * right size. Also assumes that the path ends with a "/.". The length of the + * expanded path is returned, which includes space at the end for either a + * PATH_SEPARATOR or null terminator. + */ +static ssize_t wildcard_expandPath(const char* path, char* expanded) +{ + struct dirent* file; + char* dest = expanded; + ssize_t length = 0; + size_t pathLength = strlen(path); + DIR* dir; + + dir = opendir(path); + if (dir != NULL) { + // can open dir so try to match with all *.jar and *.JAR entries + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("wildcard_expandPath: %s\n", path); +#endif + + errno = 0; + while ((file = readdir(dir)) != NULL) { + const char* filename = file->d_name; + const size_t filenameLength = strlen(filename); + const char* jarExtension; + + // If filename is smaller than 4 characters then it can not possibly + // have extension ".jar" or ".JAR" + if (filenameLength < 4) { + continue; + } + + jarExtension = &filename[filenameLength-4]; + if ((strcmp(jarExtension, ".jar") == 0) || + (strcmp(jarExtension, ".JAR") == 0)) { + + // pathLength includes an extra '.' which we'll use for either + // separator or null termination + length += pathLength + filenameLength; + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("wildcard_scanPath:\t%s\t:\t%zd\n", filename, length); +#endif + + if (expanded != NULL) { + // pathLength includes an extra '.' + strncpy(dest, path, pathLength-1); + dest += pathLength - 1; + strncpy(dest, filename, filenameLength); + dest += filenameLength; + *dest = PATH_SEPARATOR; + dest++; + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("wildcard_expandPath:\t%s\t:\t%s\n", + filename, expanded); +#endif + } + } + } + + if (errno != 0) { + fprintf(stderr, "wildcard_expandPath: on readdir %s: %s\n", + path, strerror(errno)); + length = -1; + } + + if (closedir(dir) != 0) { + fprintf(stderr, "wildcard_expandPath: on closedir %s: %s\n", + path, strerror(errno)); + } + } else if ((errno != EACCES) && (errno != ENOENT) && (errno != ENOTDIR)) { + // can not opendir due to an error we can not handle + fprintf(stderr, "wildcard_expandPath: on opendir %s: %s\n", path, + strerror(errno)); + length = -1; + } + + if (length == 0) { + // either we failed to open dir due to EACCESS, ENOENT, or ENOTDIR, or + // we did not find any file that matches *.jar or *.JAR + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + fprintf(stderr, "wildcard_expandPath: can not expand %.*s*: %s\n", + (int)(pathLength-1), path, strerror(errno)); +#endif + + // in this case, the wildcard expansion is the same as the original + // +1 for PATH_SEPARTOR or null termination + length = pathLength + 1; + if (expanded != NULL) { + // pathLength includes an extra '.' + strncpy(dest, path, pathLength-1); + dest += pathLength-1; + *dest = '*'; // restore wildcard + dest++; + *dest = PATH_SEPARATOR; + dest++; + } + } + + return length; +} + +/** + * Helper to expand classpaths. Returns the total length of the expanded + * classpath. If expandedClasspath is not NULL, then fills that with the + * expanded classpath. It assumes that expandedClasspath is of correct size, eg + * allocated after using this function with expandedClasspath=NULL to get the + * right size. + */ +static ssize_t getClassPath_helper(const char *classpath, char* expandedClasspath) +{ + ssize_t length; + ssize_t retval; + char* expandedCP_curr; + char* cp_token; + char* classpath_dup; + + classpath_dup = strdup(classpath); + if (classpath_dup == NULL) { + fprintf(stderr, "getClassPath_helper: failed strdup: %s\n", + strerror(errno)); + return -1; + } + + length = 0; + + // expandedCP_curr is the current pointer + expandedCP_curr = expandedClasspath; + + cp_token = strtok(classpath_dup, PATH_SEPARATOR_STR); + while (cp_token != NULL) { + size_t tokenlen; + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("%s\n", cp_token); +#endif + + tokenlen = strlen(cp_token); + // We only expand if token ends with "/*" + if ((tokenlen > 1) && + (cp_token[tokenlen-1] == '*') && (cp_token[tokenlen-2] == '/')) { + // replace the '*' with '.' so that we don't have to allocate another + // string for passing to opendir() in wildcard_expandPath() + cp_token[tokenlen-1] = '.'; + retval = wildcard_expandPath(cp_token, expandedCP_curr); + if (retval < 0) { + free(classpath_dup); + return -1; + } + + length += retval; + if (expandedCP_curr != NULL) { + expandedCP_curr += retval; + } + } else { + // +1 for path separator or null terminator + length += tokenlen + 1; + if (expandedCP_curr != NULL) { + strncpy(expandedCP_curr, cp_token, tokenlen); + expandedCP_curr += tokenlen; + *expandedCP_curr = PATH_SEPARATOR; + expandedCP_curr++; + } + } + + cp_token = strtok(NULL, PATH_SEPARATOR_STR); + } + + // Fix the last ':' and use it to null terminate + if (expandedCP_curr != NULL) { + expandedCP_curr--; + *expandedCP_curr = '\0'; + } + + free(classpath_dup); + return length; +} + +/** + * Gets the classpath. Wild card entries are resolved only if the entry ends + * with "/\*" (backslash to escape commenting) to match against .jar and .JAR. + * All other wild card entries (eg /path/to/dir/\*foo*) are not resolved, + * following JAVA default behavior, see: + * https://docs.oracle.com/javase/8/docs/technotes/tools/unix/classpath.html + */ +static char* getClassPath() +{ + char* classpath; + char* expandedClasspath; + ssize_t length; + ssize_t retval; + + classpath = getenv("CLASSPATH"); + if (classpath == NULL) { + return NULL; + } + + // First, get the total size of the string we will need for the expanded + // classpath + length = getClassPath_helper(classpath, NULL); + if (length < 0) { + return NULL; + } + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("+++++++++++++++++\n"); +#endif + + // we don't have to do anything if classpath has no valid wildcards + // we get length = 0 when CLASSPATH is set but empty + // if CLASSPATH is not empty, then length includes null terminator + // if length of expansion is same as original, then return a duplicate of + // original since expansion can only be longer + if ((length == 0) || ((length - 1) == strlen(classpath))) { + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + if ((length == 0) && (strlen(classpath) != 0)) { + fprintf(stderr, "Something went wrong with getting the wildcard \ + expansion length\n" ); + } +#endif + + expandedClasspath = strdup(classpath); + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("Expanded classpath=%s\n", expandedClasspath); +#endif + + return expandedClasspath; + } + + // Allocte memory for expanded classpath string + expandedClasspath = calloc(length, sizeof(char)); + if (expandedClasspath == NULL) { + fprintf(stderr, "getClassPath: failed calloc: %s\n", strerror(errno)); + return NULL; + } + + // Actual expansion + retval = getClassPath_helper(classpath, expandedClasspath); + if (retval < 0) { + free(expandedClasspath); + return NULL; + } + + // This should not happen, but dotting i's and crossing t's + if (retval != length) { + fprintf(stderr, + "Expected classpath expansion length to be %zu but instead got %zu\n", + length, retval); + free(expandedClasspath); + return NULL; + } + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("===============\n"); + printf("Allocated %zd for expanding classpath\n", length); + printf("Used %zu for expanding classpath\n", strlen(expandedClasspath) + 1); + printf("Expanded classpath=%s\n", expandedClasspath); +#endif + + return expandedClasspath; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(void) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getClassPath(); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + free(hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + + // We use findClassAndInvokeMethod here because the jclasses in + // jclasses.h have not loaded yet + jthr = findClassAndInvokeMethod(env, NULL, STATIC, NULL, HADOOP_FS, + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "FileSystem: loadFileSystems failed"); + return NULL; + } + } else { + //Attach this thread to the VM + vm = vmBuf[0]; + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (state) return state->env; + + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (state) { + mutexUnlock(&jvmMutex); + + // Free any stale exception strings. + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = NULL; + state->lastExceptionStackTrace = NULL; + + return state->env; + } + + /* Create a ThreadLocalState for this thread */ + state = threadLocalStorageCreate(); + if (!state) { + mutexUnlock(&jvmMutex); + fprintf(stderr, "getJNIEnv: Unable to create ThreadLocalState\n"); + return NULL; + } + if (threadLocalStorageSet(state)) { + mutexUnlock(&jvmMutex); + goto fail; + } + THREAD_LOCAL_STORAGE_SET_QUICK(state); + + state->env = getGlobalJNIEnv(); + mutexUnlock(&jvmMutex); + + if (!state->env) { + goto fail; + } + + jthrowable jthr = NULL; + jthr = initCachedClasses(state->env); + if (jthr) { + printExceptionAndFree(state->env, jthr, PRINT_EXC_ALL, + "initCachedClasses failed"); + goto fail; + } + return state->env; + +fail: + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + hdfsThreadDestructor(state); + return NULL; +} + +char* getLastTLSExceptionRootCause() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionRootCause; +} + +char* getLastTLSExceptionStackTrace() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionStackTrace; +} + +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return; + } + mutexUnlock(&jvmMutex); + } + + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = (char*)rootCause; + state->lastExceptionStackTrace = (char*)stackTrace; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + JC_CONFIGURATION, "set", "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return getPendingExceptionAndClear(env); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/libhdfs/hdfs_3_3/jni_helper.h b/libhdfs/hdfs_3_3/jni_helper.h new file mode 100644 index 0000000..41d6fab --- /dev/null +++ b/libhdfs/hdfs_3_3/jni_helper.h @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include "jclasses.h" + +#include +#include + +#include +#include +#include + +#ifdef WIN32 + #define PATH_SEPARATOR ';' + #define PATH_SEPARATOR_STR ";" +#else + #define PATH_SEPARATOR ':' + #define PATH_SEPARATOR_STR ":" +#endif + +// #define _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * class: The CachedJavaClass to call the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, CachedJavaClass class, + const char *methName, const char *methSignature, ...); + +/** + * findClassAndInvokeMethod: Same as invokeMethod, but it calls FindClass on + * the given className first and then calls invokeMethod. This method exists + * mainly for test infrastructure, any production code should use + * invokeMethod. Calling FindClass repeatedly can introduce performance + * overhead, so users should prefer invokeMethod and supply a CachedJavaClass. + */ +jthrowable findClassAndInvokeMethod(JNIEnv *env, jvalue *retval, + MethType methType, jobject instObj, const char *className, + const char *methName, const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, + const char *className, const char *ctorSignature, ...); + +/** + * Same as constructNewObjectOfClass but it takes in a CachedJavaClass + * rather than a className. This avoids an extra call to FindClass. + */ +jthrowable constructNewObjectOfCachedClass(JNIEnv *env, jobject *out, + CachedJavaClass cachedJavaClass, const char *ctorSignature, ...); + +jthrowable methodIdFromClass(jclass cls, const char *className, + const char *methName, const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * It gets this from the ThreadLocalState if it exists. If a ThreadLocalState + * does not exist, one will be created. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Get the last exception root cause that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The root cause as a C-string. + */ +char* getLastTLSExceptionRootCause(); + +/** + * Get the last exception stack trace that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The stack trace as a C-string. + */ +char* getLastTLSExceptionStackTrace(); + +/** setTLSExceptionStrings: Sets the 'rootCause' and 'stackTrace' in the + * ThreadLocalState if one exists for the current thread. + * + * @param rootCause A string containing the root cause of an exception. + * @param stackTrace A string containing the stack trace of an exception. + * @return None. + */ +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/libhdfs/hdfs_3_3/os/mutexes.h b/libhdfs/hdfs_3_3/os/mutexes.h new file mode 100644 index 0000000..92afabd --- /dev/null +++ b/libhdfs/hdfs_3_3/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** Mutex protecting initialization of jclasses in jclasses.h. */ +extern mutex jclassInitMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/libhdfs/hdfs_3_3/os/posix/mutexes.c b/libhdfs/hdfs_3_3/os/posix/mutexes.c new file mode 100644 index 0000000..5c6b429 --- /dev/null +++ b/libhdfs/hdfs_3_3/os/posix/mutexes.c @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex jvmMutex; +mutex jclassInitMutex = PTHREAD_MUTEX_INITIALIZER; +pthread_mutexattr_t jvmMutexAttr; + +__attribute__((constructor)) static void init() { + pthread_mutexattr_init(&jvmMutexAttr); + pthread_mutexattr_settype(&jvmMutexAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&jvmMutex, &jvmMutexAttr); +} + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/libhdfs/hdfs_3_3/os/posix/platform.h b/libhdfs/hdfs_3_3/os/posix/platform.h new file mode 100644 index 0000000..c63bbf9 --- /dev/null +++ b/libhdfs/hdfs_3_3/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/libhdfs/hdfs_3_3/os/posix/thread.c b/libhdfs/hdfs_3_3/os/posix/thread.c new file mode 100644 index 0000000..af0c61f --- /dev/null +++ b/libhdfs/hdfs_3_3/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/libhdfs/hdfs_3_3/os/posix/thread_local_storage.c b/libhdfs/hdfs_3_3/os/posix/thread_local_storage.c new file mode 100644 index 0000000..1b6dafa --- /dev/null +++ b/libhdfs/hdfs_3_3/os/posix/thread_local_storage.c @@ -0,0 +1,193 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +#include "exception.h" +#include "jni_helper.h" + +#define UNKNOWN "UNKNOWN" +#define MAXTHRID 256 + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +static void get_current_thread_id(JNIEnv* env, char* id, int max); + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + struct ThreadLocalState *state = (struct ThreadLocalState*)v; + JNIEnv *env = state->env;; + jint ret; + jthrowable jthr; + char thr_name[MAXTHRID]; + + /* Detach the current thread from the JVM */ + if ((env != NULL) && (*env != NULL)) { + ret = (*env)->GetJavaVM(env, &vm); + + if (ret != 0) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionDescribe(env); + (*env)->ExceptionClear(env); + } + } else { + ret = (*vm)->DetachCurrentThread(vm); + + if (ret != JNI_OK) { + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionDescribe(env); + (*env)->ExceptionClear(env); + } + get_current_thread_id(env, thr_name, MAXTHRID); + + fprintf(stderr, "hdfsThreadDestructor: Unable to detach thread %s " + "from the JVM. Error code: %d\n", thr_name, ret); + } + } + } + + /* Free exception strings */ + if (state->lastExceptionStackTrace) free(state->lastExceptionStackTrace); + if (state->lastExceptionRootCause) free(state->lastExceptionRootCause); + + /* Free the state itself */ + free(state); +} + +static void get_current_thread_id(JNIEnv* env, char* id, int max) { + jvalue jVal; + jobject thr = NULL; + jstring thr_name = NULL; + jlong thr_id = 0; + jthrowable jthr = NULL; + const char *thr_name_str; + + jthr = findClassAndInvokeMethod(env, &jVal, STATIC, NULL, "java/lang/Thread", + "currentThread", "()Ljava/lang/Thread;"); + if (jthr) { + snprintf(id, max, "%s", UNKNOWN); + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "get_current_thread_id: Thread#currentThread failed: "); + goto done; + } + thr = jVal.l; + + jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, thr, + "java/lang/Thread", "getId", "()J"); + if (jthr) { + snprintf(id, max, "%s", UNKNOWN); + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "get_current_thread_id: Thread#getId failed: "); + goto done; + } + thr_id = jVal.j; + + jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, thr, + "java/lang/Thread", "toString", "()Ljava/lang/String;"); + if (jthr) { + snprintf(id, max, "%s:%ld", UNKNOWN, thr_id); + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "get_current_thread_id: Thread#toString failed: "); + goto done; + } + thr_name = jVal.l; + + thr_name_str = (*env)->GetStringUTFChars(env, thr_name, NULL); + if (!thr_name_str) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "get_current_thread_id: GetStringUTFChars failed: "); + snprintf(id, max, "%s:%ld", UNKNOWN, thr_id); + goto done; + } + + // Treating the jlong as a long *should* be safe + snprintf(id, max, "%s:%ld", thr_name_str, thr_id); + + // Release the char* + (*env)->ReleaseStringUTFChars(env, thr_name, thr_name_str); + +done: + destroyLocalReference(env, thr); + destroyLocalReference(env, thr_name); + + // Make sure the id is null terminated in case we overflow the max length + id[max - 1] = '\0'; +} + +struct ThreadLocalState* threadLocalStorageCreate() +{ + struct ThreadLocalState *state; + state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState)); + if (state == NULL) { + fprintf(stderr, + "threadLocalStorageCreate: OOM - Unable to allocate thread local state\n"); + return NULL; + } + state->lastExceptionStackTrace = NULL; + state->lastExceptionRootCause = NULL; + return state; +} + +int threadLocalStorageGet(struct ThreadLocalState **state) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *state = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(struct ThreadLocalState *state) +{ + int ret = pthread_setspecific(gTlsKey, state); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(state); + } + return ret; +} diff --git a/libhdfs/hdfs_3_3/os/thread.h b/libhdfs/hdfs_3_3/os/thread.h new file mode 100644 index 0000000..ae425d3 --- /dev/null +++ b/libhdfs/hdfs_3_3/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/libhdfs/hdfs_3_3/os/thread_local_storage.h b/libhdfs/hdfs_3_3/os/thread_local_storage.h new file mode 100644 index 0000000..025ceff --- /dev/null +++ b/libhdfs/hdfs_3_3/os/thread_local_storage.h @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) \ + static __thread struct ThreadLocalState *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + *state = quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) \ + { \ + quickTlsEnv = (state); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) +#endif + +struct ThreadLocalState { + /* The JNIEnv associated with the current thread */ + JNIEnv *env; + /* The last exception stack trace that occured on this thread */ + char *lastExceptionStackTrace; + /* The last exception root cause that occured on this thread */ + char *lastExceptionRootCause; +}; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v); + +/** + * Creates an object of ThreadLocalState. + * + * @return The newly created object if successful, NULL otherwise. + */ +struct ThreadLocalState* threadLocalStorageCreate(); + +/** + * Gets the ThreadLocalState in thread-local storage for the current thread. + * If the call succeeds, and there is a ThreadLocalState associated with this + * thread, then returns 0 and populates 'state'. If the call succeeds, but + * there is no ThreadLocalState associated with this thread, then returns 0 + * and sets ThreadLocalState to NULL. If the call fails, then returns non-zero. + * Only one thread at a time may execute this function. The caller is + * responsible for enforcing mutual exclusion. + * + * @param env ThreadLocalState out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(struct ThreadLocalState **state); + +/** + * Sets the ThreadLocalState in thread-local storage for the current thread. + * + * @param env ThreadLocalState to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(struct ThreadLocalState *state); + +#endif diff --git a/libhdfs/hdfs_3_3/os/windows/inttypes.h b/libhdfs/hdfs_3_3/os/windows/inttypes.h new file mode 100644 index 0000000..a520d15 --- /dev/null +++ b/libhdfs/hdfs_3_3/os/windows/inttypes.h @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_INTTYPES_H +#define LIBHDFS_INTTYPES_H + +/* On Windows, inttypes.h does not exist, so manually define what we need. */ + +#define PRId64 "I64d" +#define PRIu64 "I64u" +typedef unsigned __int64 uint64_t; + +#endif diff --git a/libhdfs/hdfs_3_3/os/windows/mutexes.c b/libhdfs/hdfs_3_3/os/windows/mutexes.c new file mode 100644 index 0000000..ac7f9fd --- /dev/null +++ b/libhdfs/hdfs_3_3/os/windows/mutexes.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include + +mutex jvmMutex; +mutex jclassInitMutex; + +/** + * Unfortunately, there is no simple static initializer for a critical section. + * Instead, the API requires calling InitializeCriticalSection. Since libhdfs + * lacks an explicit initialization function, there is no obvious existing place + * for the InitializeCriticalSection calls. To work around this, we define an + * initialization function and instruct the linker to set a pointer to that + * function as a user-defined global initializer. See discussion of CRT + * Initialization: + * http://msdn.microsoft.com/en-us/library/bb918180.aspx + */ +static void __cdecl initializeMutexes(void) { + InitializeCriticalSection(&jvmMutex); + InitializeCriticalSection(&jclassInitMutex); +} +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) +const void (__cdecl *pInitialize)(void) = initializeMutexes; + +int mutexLock(mutex *m) { + EnterCriticalSection(m); + return 0; +} + +int mutexUnlock(mutex *m) { + LeaveCriticalSection(m); + return 0; +} diff --git a/libhdfs/hdfs_3_3/os/windows/platform.h b/libhdfs/hdfs_3_3/os/windows/platform.h new file mode 100644 index 0000000..9eedfde --- /dev/null +++ b/libhdfs/hdfs_3_3/os/windows/platform.h @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include +#include +#include + +/* + * O_ACCMODE defined to match Linux definition. + */ +#ifndef O_ACCMODE +#define O_ACCMODE 0x0003 +#endif + +/* + * Windows has a different name for its maximum path length constant. + */ +#ifndef PATH_MAX +#define PATH_MAX MAX_PATH +#endif + +/* + * Windows does not define EDQUOT and ESTALE in errno.h. The closest equivalents + * are these constants from winsock.h. + */ +#ifndef EDQUOT +#define EDQUOT WSAEDQUOT +#endif + +#ifndef ESTALE +#define ESTALE WSAESTALE +#endif + +/* + * gcc-style type-checked format arguments are not supported on Windows, so just + * stub this macro. + */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) + +/* + * Define macros for various string formatting functions not defined on Windows. + * Where possible, we reroute to one of the secure CRT variants. On Windows, + * the preprocessor does support variadic macros, even though they weren't + * defined until C99. + */ +#define snprintf(str, size, format, ...) \ + _snprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) +#define strncpy(dest, src, n) \ + strncpy_s((dest), (n), (src), _TRUNCATE) +#define strtok_r(str, delim, saveptr) \ + strtok_s((str), (delim), (saveptr)) +#define vsnprintf(str, size, format, ...) \ + vsnprintf_s((str), (size), _TRUNCATE, (format), __VA_ARGS__) + +/* + * Mutex data type defined as Windows CRITICAL_SECTION. A critical section (not + * Windows mutex) is used, because libhdfs only needs synchronization of multiple + * threads within a single process, not synchronization across process + * boundaries. + */ +typedef CRITICAL_SECTION mutex; + +/* + * Thread data type defined as HANDLE to a Windows thread. + */ +typedef HANDLE threadId; + +#endif diff --git a/libhdfs/hdfs_3_3/os/windows/thread.c b/libhdfs/hdfs_3_3/os/windows/thread.c new file mode 100644 index 0000000..f5cc2a7 --- /dev/null +++ b/libhdfs/hdfs_3_3/os/windows/thread.c @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by CreateThread. + * + * @param toRun thread to run + * @return DWORD result of running thread (always 0) + */ +static DWORD WINAPI runThread(LPVOID toRun) { + const thread *t = toRun; + t->start(t->arg); + return 0; +} + +int threadCreate(thread *t) { + DWORD ret = 0; + HANDLE h; + h = CreateThread(NULL, 0, runThread, t, 0, NULL); + if (h) { + t->id = h; + } else { + ret = GetLastError(); + fprintf(stderr, "threadCreate: CreateThread failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + DWORD ret = WaitForSingleObject(t->id, INFINITE); + switch (ret) { + case WAIT_OBJECT_0: + break; + case WAIT_FAILED: + ret = GetLastError(); + fprintf(stderr, "threadJoin: WaitForSingleObject failed with error %d\n", + ret); + break; + default: + fprintf(stderr, "threadJoin: WaitForSingleObject unexpected error %d\n", + ret); + break; + } + return ret; +} diff --git a/libhdfs/hdfs_3_3/os/windows/thread_local_storage.c b/libhdfs/hdfs_3_3/os/windows/thread_local_storage.c new file mode 100644 index 0000000..f7abc89 --- /dev/null +++ b/libhdfs/hdfs_3_3/os/windows/thread_local_storage.c @@ -0,0 +1,265 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +#define UNKNOWN "UNKNOWN" +#define MAXTHRID 256 + +/** Key that allows us to retrieve thread-local storage */ +static DWORD gTlsIndex = TLS_OUT_OF_INDEXES; + +static void get_current_thread_id(JNIEnv* env, char* id, int max); + +/** + * If the current thread has a JNIEnv in thread-local storage, then detaches the + * current thread from the JVM and also frees up the ThreadLocalState object. + */ +static void detachCurrentThreadFromJvm() +{ + struct ThreadLocalState *state = NULL; + JNIEnv *env = NULL; + JavaVM *vm; + jint ret; + char thr_name[MAXTHRID]; + + if (threadLocalStorageGet(&state) || !state) { + return; + } + env = state->env; + if ((env == NULL) || (*env == NULL)) { + return; + } + ret = (*env)->GetJavaVM(env, &vm); + if (ret) { + fprintf(stderr, + "detachCurrentThreadFromJvm: GetJavaVM failed with error %d\n", + ret); + (*env)->ExceptionDescribe(env); + } else { + ret = (*vm)->DetachCurrentThread(vm); + + if (ret != JNI_OK) { + (*env)->ExceptionDescribe(env); + get_current_thread_id(env, thr_name, MAXTHRID); + + fprintf(stderr, "detachCurrentThreadFromJvm: Unable to detach thread %s " + "from the JVM. Error code: %d\n", thr_name, ret); + } + } + + /* Free exception strings */ + if (state->lastExceptionStackTrace) free(state->lastExceptionStackTrace); + if (state->lastExceptionRootCause) free(state->lastExceptionRootCause); + + /* Free the state itself */ + free(state); +} + +static void get_current_thread_id(JNIEnv* env, char* id, int max) { + jclass cls; + jmethodID mth; + jobject thr; + jstring thr_name; + jlong thr_id = 0; + const char *thr_name_str; + + cls = (*env)->FindClass(env, "java/lang/Thread"); + mth = (*env)->GetStaticMethodID(env, cls, "currentThread", + "()Ljava/lang/Thread;"); + thr = (*env)->CallStaticObjectMethod(env, cls, mth); + + if (thr != NULL) { + mth = (*env)->GetMethodID(env, cls, "getId", "()J"); + thr_id = (*env)->CallLongMethod(env, thr, mth); + (*env)->ExceptionDescribe(env); + + mth = (*env)->GetMethodID(env, cls, "toString", "()Ljava/lang/String;"); + thr_name = (jstring)(*env)->CallObjectMethod(env, thr, mth); + + if (thr_name != NULL) { + thr_name_str = (*env)->GetStringUTFChars(env, thr_name, NULL); + + // Treating the jlong as a long *should* be safe + snprintf(id, max, "%s:%ld", thr_name_str, thr_id); + + // Release the char* + (*env)->ReleaseStringUTFChars(env, thr_name, thr_name_str); + } else { + (*env)->ExceptionDescribe(env); + + // Treating the jlong as a long *should* be safe + snprintf(id, max, "%s:%ld", UNKNOWN, thr_id); + } + } else { + (*env)->ExceptionDescribe(env); + snprintf(id, max, "%s", UNKNOWN); + } + + // Make sure the id is null terminated in case we overflow the max length + id[max - 1] = '\0'; +} + +void hdfsThreadDestructor(void *v) +{ + // Ignore 'v' since it will contain the state and we will obtain it in the below + // call anyway. + detachCurrentThreadFromJvm(); +} + +/** + * Unlike pthreads, the Windows API does not seem to provide a convenient way to + * hook a callback onto thread shutdown. However, the Windows portable + * executable format does define a concept of thread-local storage callbacks. + * Here, we define a function and instruct the linker to set a pointer to that + * function in the segment for thread-local storage callbacks. See page 85 of + * Microsoft Portable Executable and Common Object File Format Specification: + * http://msdn.microsoft.com/en-us/gg463119.aspx + * This technique only works for implicit linking (OS loads DLL on demand), not + * for explicit linking (user code calls LoadLibrary directly). This effectively + * means that we have a known limitation: libhdfs may not work correctly if a + * Windows application attempts to use it via explicit linking. + * + * @param h module handle + * @param reason the reason for calling the callback + * @param pv reserved, unused + */ +static void NTAPI tlsCallback(PVOID h, DWORD reason, PVOID pv) +{ + DWORD tlsIndex; + switch (reason) { + case DLL_THREAD_DETACH: + detachCurrentThreadFromJvm(); + break; + case DLL_PROCESS_DETACH: + detachCurrentThreadFromJvm(); + tlsIndex = gTlsIndex; + gTlsIndex = TLS_OUT_OF_INDEXES; + if (!TlsFree(tlsIndex)) { + fprintf(stderr, "tlsCallback: TlsFree failed with error %d\n", + GetLastError()); + } + break; + default: + break; + } +} + +/* + * A variable named _tls_used contains the TLS directory, which contains a list + * of pointers to callback functions. Normally, the linker won't retain this + * variable unless the executable has implicit thread-local variables, defined + * using the __declspec(thread) extended storage-class modifier. libhdfs + * doesn't use __declspec(thread), and we have no guarantee that the executable + * linked to libhdfs will use __declspec(thread). By forcing the linker to + * reference _tls_used, we guarantee that the binary retains the TLS directory. + * See Microsoft Visual Studio 10.0/VC/crt/src/tlssup.c . + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#endif + +/* + * We must retain a pointer to the callback function. Force the linker to keep + * this symbol, even though it appears that nothing in our source code uses it. + */ +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:pTlsCallback") +#else +#pragma comment(linker, "/INCLUDE:_pTlsCallback") +#endif + +/* + * Define constant pointer to our callback, and tell the linker to pin it into + * the TLS directory so that it receives thread callbacks. Use external linkage + * to protect against the linker discarding the seemingly unused symbol. + */ +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK pTlsCallback; +const PIMAGE_TLS_CALLBACK pTlsCallback = tlsCallback; +#pragma const_seg() + +struct ThreadLocalState* threadLocalStorageCreate() +{ + struct ThreadLocalState *state; + state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState)); + if (state == NULL) { + fprintf(stderr, + "threadLocalStorageCreate: OOM - Unable to allocate thread local state\n"); + return NULL; + } + state->lastExceptionStackTrace = NULL; + state->lastExceptionRootCause = NULL; + return state; +} + +int threadLocalStorageGet(struct ThreadLocalState **state) +{ + LPVOID tls; + DWORD ret; + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + gTlsIndex = TlsAlloc(); + if (TLS_OUT_OF_INDEXES == gTlsIndex) { + fprintf(stderr, + "threadLocalStorageGet: TlsAlloc failed with error %d\n", + TLS_OUT_OF_INDEXES); + return TLS_OUT_OF_INDEXES; + } + } + tls = TlsGetValue(gTlsIndex); + if (tls) { + *state = tls; + return 0; + } else { + ret = GetLastError(); + if (ERROR_SUCCESS == ret) { + /* Thread-local storage contains NULL, because we haven't set it yet. */ + *state = NULL; + return 0; + } else { + /* + * The API call failed. According to documentation, TlsGetValue cannot + * fail as long as the index is a valid index from a successful TlsAlloc + * call. This error handling is purely defensive. + */ + fprintf(stderr, + "threadLocalStorageGet: TlsGetValue failed with error %d\n", ret); + return ret; + } + } +} + +int threadLocalStorageSet(struct ThreadLocalState *state) +{ + DWORD ret = 0; + if (!TlsSetValue(gTlsIndex, (LPVOID)state)) { + ret = GetLastError(); + fprintf(stderr, + "threadLocalStorageSet: TlsSetValue failed with error %d\n", + ret); + detachCurrentThreadFromJvm(state); + } + return ret; +} diff --git a/libhdfs/hdfs_3_3/os/windows/unistd.h b/libhdfs/hdfs_3_3/os/windows/unistd.h new file mode 100644 index 0000000..b82ce48 --- /dev/null +++ b/libhdfs/hdfs_3_3/os/windows/unistd.h @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_UNISTD_H +#define LIBHDFS_UNISTD_H + +/* On Windows, unistd.h does not exist, so manually define what we need. */ + +#include /* Declares getpid(). */ +#include + +/* Re-route sleep to Sleep, converting units from seconds to milliseconds. */ +#define sleep(seconds) Sleep((seconds) * 1000) +#endif From 8a108c752784aab83d9cef414de15bdcdda94e71 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 14:19:54 +0800 Subject: [PATCH 02/20] Don't need to check three hdfs version Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30ed795..08398a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,6 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - hdfs-version: [ "2.10.1", "3.2.3", "3.3.2" ] os: [ ubuntu-latest ] steps: From 9d3eab8d8a3db86de9072da441b95f4304acffae Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 14:22:28 +0800 Subject: [PATCH 03/20] Fix build on macos Signed-off-by: Xuanwo --- build.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/build.rs b/build.rs index 27d9df1..fb716af 100644 --- a/build.rs +++ b/build.rs @@ -19,12 +19,16 @@ fn main() -> Result<(), Box> { builder.static_flag(true); builder.static_crt(true); - { - builder.include(format!("{java_home}/include")); + // Handle java headers. + builder.include(format!("{java_home}/include")); + if cfg!(os = "linux") { builder.include(format!("{java_home}/include/linux")); } + if cfg!(os = "macos") { + builder.include(format!("{java_home}/include/macos")); + } - // Choose the latest version. + // Choose the latest hdfs version. let mut version = "hdfs_2_2"; if cfg!(feature = "hdfs_2_3") { version = "hdfs_2_3" From 464546b144d321cc3ac61c7d0e7b95557f0de1e6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 14:28:21 +0800 Subject: [PATCH 04/20] Fix build Signed-off-by: Xuanwo --- build.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.rs b/build.rs index fb716af..cf4fd76 100644 --- a/build.rs +++ b/build.rs @@ -21,10 +21,10 @@ fn main() -> Result<(), Box> { // Handle java headers. builder.include(format!("{java_home}/include")); - if cfg!(os = "linux") { + if cfg!(target_os = "linux") { builder.include(format!("{java_home}/include/linux")); } - if cfg!(os = "macos") { + if cfg!(target_os = "macos") { builder.include(format!("{java_home}/include/macos")); } @@ -77,7 +77,7 @@ fn main() -> Result<(), Box> { if cfg!(feature = "hdfs_2_6") { builder.include(format!("libhdfs/{version}/os")); - if cfg!(os = "windows") { + if cfg!(windows) { builder.include(format!("libhdfs/{version}/os/windows")); builder.file(format!("libhdfs/{version}/os/windows/mutexes.c")); builder.file(format!("libhdfs/{version}/os/windows/thread.c")); From 117896d28aacdeae451495da75949be7e65a8c1c Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 14:31:19 +0800 Subject: [PATCH 05/20] Fix build on mac Signed-off-by: Xuanwo --- build.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.rs b/build.rs index cf4fd76..a24b3be 100644 --- a/build.rs +++ b/build.rs @@ -25,7 +25,10 @@ fn main() -> Result<(), Box> { builder.include(format!("{java_home}/include/linux")); } if cfg!(target_os = "macos") { - builder.include(format!("{java_home}/include/macos")); + builder.include(format!("{java_home}/include/darwin")); + } + if cfg!(target_os = "windows") { + builder.include(format!("{java_home}/include/win32")); } // Choose the latest hdfs version. From 40970897760d4e8fff82ea826af9bdac2dd1fac6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 14:46:06 +0800 Subject: [PATCH 06/20] Fix build on windows Signed-off-by: Xuanwo --- build.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/build.rs b/build.rs index a24b3be..b4ba3f1 100644 --- a/build.rs +++ b/build.rs @@ -9,6 +9,7 @@ fn main() -> Result<(), Box> { // Make sure jvm has been linked. let java_home = env::var("JAVA_HOME")?; println!("cargo:rustc-link-search=native={java_home}/lib/server"); + println!("cargo:rustc-link-search=native={java_home}/lib/amd64/server"); println!("cargo:rustc-link-lib=jvm"); // Static link compiled `libhdfs.a` @@ -16,6 +17,7 @@ fn main() -> Result<(), Box> { let mut builder = cc::Build::new(); builder.warnings(false); + // builder.flag_if_supported("-Wno-incompatible-pointer-types"); builder.static_flag(true); builder.static_crt(true); @@ -42,6 +44,13 @@ fn main() -> Result<(), Box> { if cfg!(feature = "hdfs_2_5") { version = "hdfs_2_5" } + // Since 2.6, hdfs supports windows. + // + // We build with src from `hdfs_2_6` but expose earlier ABI like `hdfs_2_2`. + // This simple trick makes hdfs-sys works on windows without breaking our ABI promise. + if cfg!(target_os = "windows") { + version = "hdfs_2_6" + } if cfg!(feature = "hdfs_2_6") { version = "hdfs_2_6" } @@ -80,7 +89,7 @@ fn main() -> Result<(), Box> { if cfg!(feature = "hdfs_2_6") { builder.include(format!("libhdfs/{version}/os")); - if cfg!(windows) { + if cfg!(target_os = "windows") { builder.include(format!("libhdfs/{version}/os/windows")); builder.file(format!("libhdfs/{version}/os/windows/mutexes.c")); builder.file(format!("libhdfs/{version}/os/windows/thread.c")); From 5d3b851e634e5e1d5c3c0710ff668a80e93bfdc6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 15:03:25 +0800 Subject: [PATCH 07/20] Use C11 instead Signed-off-by: Xuanwo --- build.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/build.rs b/build.rs index b4ba3f1..aa5e740 100644 --- a/build.rs +++ b/build.rs @@ -17,10 +17,19 @@ fn main() -> Result<(), Box> { let mut builder = cc::Build::new(); builder.warnings(false); - // builder.flag_if_supported("-Wno-incompatible-pointer-types"); builder.static_flag(true); builder.static_crt(true); + // Ignore all warnings from cc as we don't care about code written by Apache Hadoop. + builder.flag_if_supported("-w"); + + // Use c++ 14 instead. + if cfg!(windows) { + builder.flag("/std:c11"); + } else { + builder.flag("--std=c11"); + } + // Handle java headers. builder.include(format!("{java_home}/include")); if cfg!(target_os = "linux") { @@ -86,7 +95,7 @@ fn main() -> Result<(), Box> { builder.file(format!("libhdfs/{version}/hdfs.c")); // Since 2.6, we need to include mutexes. - if cfg!(feature = "hdfs_2_6") { + if cfg!(feature = "hdfs_2_6") || cfg!(target_os = "windows") { builder.include(format!("libhdfs/{version}/os")); if cfg!(target_os = "windows") { From d544e3f65b09c5d14657a3b3b1f750cdd26e71b4 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 15:16:48 +0800 Subject: [PATCH 08/20] Disable c11 Signed-off-by: Xuanwo --- build.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/build.rs b/build.rs index aa5e740..c3a3e82 100644 --- a/build.rs +++ b/build.rs @@ -23,13 +23,6 @@ fn main() -> Result<(), Box> { // Ignore all warnings from cc as we don't care about code written by Apache Hadoop. builder.flag_if_supported("-w"); - // Use c++ 14 instead. - if cfg!(windows) { - builder.flag("/std:c11"); - } else { - builder.flag("--std=c11"); - } - // Handle java headers. builder.include(format!("{java_home}/include")); if cfg!(target_os = "linux") { From d13fa3ac60dbf9920359c1fcadfb6bad03b9b08b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 15:32:30 +0800 Subject: [PATCH 09/20] Hack windows Signed-off-by: Xuanwo --- build.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/build.rs b/build.rs index c3a3e82..7bc4849 100644 --- a/build.rs +++ b/build.rs @@ -22,6 +22,31 @@ fn main() -> Result<(), Box> { // Ignore all warnings from cc as we don't care about code written by Apache Hadoop. builder.flag_if_supported("-w"); + builder.flag_if_supported("-std=c++17"); + + // Inspired by [hadoop-hdfs-native-client/src/CMakeLists.txt](https://github.com/apache/hadoop/blob/trunk/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt) + if cfg!(windows) { + // Set the optimizer level. + builder.flag("-O2"); + // Set warning level 4. + builder.file("/W4"); + // Skip "unreferenced formal parameter". + builder.flag("/wd4100"); + // Skip "conditional expression is constant". + builder.flag("/wd4127"); + // Skip deprecated POSIX function warnings. + builder.flag("-D_CRT_NONSTDC_NO_DEPRECATE"); + // Skip CRT non-secure function warnings. If we can convert usage of + // strerror, getenv and ctime to their secure CRT equivalents, then we can + // re-enable the CRT non-secure function warnings. + builder.flag("-D_CRT_SECURE_NO_WARNINGS"); + // Omit unneeded headers. + builder.flag("-DWIN32_LEAN_AND_MEAN"); + } else { + builder.flag("-fvisibility=hidden"); + // using old default behavior on GCC >= 10.0 + builder.flag("-fcommon"); + } // Handle java headers. builder.include(format!("{java_home}/include")); From 9ae54b299a3e1bac0af3a5c94f42c3c217a3d9e1 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 15:38:04 +0800 Subject: [PATCH 10/20] Fix typo Signed-off-by: Xuanwo --- build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.rs b/build.rs index 7bc4849..25c20c6 100644 --- a/build.rs +++ b/build.rs @@ -29,7 +29,7 @@ fn main() -> Result<(), Box> { // Set the optimizer level. builder.flag("-O2"); // Set warning level 4. - builder.file("/W4"); + builder.flag("/W4"); // Skip "unreferenced formal parameter". builder.flag("/wd4100"); // Skip "conditional expression is constant". From 8cf8a0607aa6278e94f34d27d6c4dd224890f0f2 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 15:43:05 +0800 Subject: [PATCH 11/20] Print java home for debug Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08398a2..dd5631f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: steps: - uses: actions/checkout@v3 - + - run: tree ${{ env.JAVA_HOME }} - name: Test uses: actions-rs/cargo@v1 with: From a2d4f5130208c31e6ce26cde500efce10d75dbad Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 15:52:32 +0800 Subject: [PATCH 12/20] Fix java on macos Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd5631f..10e5b3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,10 @@ jobs: steps: - uses: actions/checkout@v3 - - run: tree ${{ env.JAVA_HOME }} + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' - name: Test uses: actions-rs/cargo@v1 with: From 3959217c6c8877119325d7915bde2c4739bdcf34 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 16:10:33 +0800 Subject: [PATCH 13/20] Workaround hdfs 3_3 Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 1 - build.rs | 6 + dirent/include/dirent.h | 1028 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 1034 insertions(+), 1 deletion(-) create mode 100644 dirent/include/dirent.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10e5b3a..0eebdc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: Swatinem/rust-cache@v1 - name: Build uses: actions-rs/cargo@v1 with: diff --git a/build.rs b/build.rs index 25c20c6..f8914d1 100644 --- a/build.rs +++ b/build.rs @@ -147,6 +147,12 @@ fn main() -> Result<(), Box> { builder.file(format!("libhdfs/{version}/jclasses.c")); } + // Sadly, hadoop include `dirent.h` but mvsc doesn't support it. + // So we include [tronkko/dirent](https://raw.githubusercontent.com/tronkko/dirent/master/include/dirent.h) instead. + if cfg!(feature = "hdfs_3_3") && cfg!(target_os = "windows") { + builder.include("dirent/include") + } + builder.compile("hdfs"); Ok(()) } diff --git a/dirent/include/dirent.h b/dirent/include/dirent.h new file mode 100644 index 0000000..ad52607 --- /dev/null +++ b/dirent/include/dirent.h @@ -0,0 +1,1028 @@ +/* + * Dirent interface for Microsoft Visual Studio + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#ifndef DIRENT_H +#define DIRENT_H + +/* Hide warnings about unreferenced local functions */ +#if defined(__clang__) +# pragma clang diagnostic ignored "-Wunused-function" +#elif defined(_MSC_VER) +# pragma warning(disable:4505) +#elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" +#endif + +/* + * Include windows.h without Windows Sockets 1.1 to prevent conflicts with + * Windows Sockets 2.0. + */ +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Indicates that d_type field is available in dirent structure */ +#define _DIRENT_HAVE_D_TYPE + +/* Indicates that d_namlen field is available in dirent structure */ +#define _DIRENT_HAVE_D_NAMLEN + +/* Entries missing from MSVC 6.0 */ +#if !defined(FILE_ATTRIBUTE_DEVICE) +# define FILE_ATTRIBUTE_DEVICE 0x40 +#endif + +/* File type and permission flags for stat(), general mask */ +#if !defined(S_IFMT) +# define S_IFMT _S_IFMT +#endif + +/* Directory bit */ +#if !defined(S_IFDIR) +# define S_IFDIR _S_IFDIR +#endif + +/* Character device bit */ +#if !defined(S_IFCHR) +# define S_IFCHR _S_IFCHR +#endif + +/* Pipe bit */ +#if !defined(S_IFFIFO) +# define S_IFFIFO _S_IFFIFO +#endif + +/* Regular file bit */ +#if !defined(S_IFREG) +# define S_IFREG _S_IFREG +#endif + +/* Read permission */ +#if !defined(S_IREAD) +# define S_IREAD _S_IREAD +#endif + +/* Write permission */ +#if !defined(S_IWRITE) +# define S_IWRITE _S_IWRITE +#endif + +/* Execute permission */ +#if !defined(S_IEXEC) +# define S_IEXEC _S_IEXEC +#endif + +/* Pipe */ +#if !defined(S_IFIFO) +# define S_IFIFO _S_IFIFO +#endif + +/* Block device */ +#if !defined(S_IFBLK) +# define S_IFBLK 0 +#endif + +/* Link */ +#if !defined(S_IFLNK) +# define S_IFLNK 0 +#endif + +/* Socket */ +#if !defined(S_IFSOCK) +# define S_IFSOCK 0 +#endif + +/* Read user permission */ +#if !defined(S_IRUSR) +# define S_IRUSR S_IREAD +#endif + +/* Write user permission */ +#if !defined(S_IWUSR) +# define S_IWUSR S_IWRITE +#endif + +/* Execute user permission */ +#if !defined(S_IXUSR) +# define S_IXUSR 0 +#endif + +/* Read group permission */ +#if !defined(S_IRGRP) +# define S_IRGRP 0 +#endif + +/* Write group permission */ +#if !defined(S_IWGRP) +# define S_IWGRP 0 +#endif + +/* Execute group permission */ +#if !defined(S_IXGRP) +# define S_IXGRP 0 +#endif + +/* Read others permission */ +#if !defined(S_IROTH) +# define S_IROTH 0 +#endif + +/* Write others permission */ +#if !defined(S_IWOTH) +# define S_IWOTH 0 +#endif + +/* Execute others permission */ +#if !defined(S_IXOTH) +# define S_IXOTH 0 +#endif + +/* Maximum length of file name */ +#if !defined(PATH_MAX) +# define PATH_MAX MAX_PATH +#endif +#if !defined(FILENAME_MAX) +# define FILENAME_MAX MAX_PATH +#endif +#if !defined(NAME_MAX) +# define NAME_MAX FILENAME_MAX +#endif + +/* File type flags for d_type */ +#define DT_UNKNOWN 0 +#define DT_REG S_IFREG +#define DT_DIR S_IFDIR +#define DT_FIFO S_IFIFO +#define DT_SOCK S_IFSOCK +#define DT_CHR S_IFCHR +#define DT_BLK S_IFBLK +#define DT_LNK S_IFLNK + +/* Macros for converting between st_mode and d_type */ +#define IFTODT(mode) ((mode) & S_IFMT) +#define DTTOIF(type) (type) + +/* + * File type macros. Note that block devices, sockets and links cannot be + * distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are + * only defined for compatibility. These macros should always return false + * on Windows. + */ +#if !defined(S_ISFIFO) +# define S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFIFO) +#endif +#if !defined(S_ISDIR) +# define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#endif +#if !defined(S_ISREG) +# define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#endif +#if !defined(S_ISLNK) +# define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) +#endif +#if !defined(S_ISSOCK) +# define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) +#endif +#if !defined(S_ISCHR) +# define S_ISCHR(mode) (((mode) & S_IFMT) == S_IFCHR) +#endif +#if !defined(S_ISBLK) +# define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) +#endif + +/* Return the exact length of the file name without zero terminator */ +#define _D_EXACT_NAMLEN(p) ((p)->d_namlen) + +/* Return the maximum size of a file name */ +#define _D_ALLOC_NAMLEN(p) ((PATH_MAX)+1) + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* Wide-character version */ +struct _wdirent { + /* Always zero */ + long d_ino; + + /* File position within stream */ + long d_off; + + /* Structure size */ + unsigned short d_reclen; + + /* Length of name without \0 */ + size_t d_namlen; + + /* File type */ + int d_type; + + /* File name */ + wchar_t d_name[PATH_MAX+1]; +}; +typedef struct _wdirent _wdirent; + +struct _WDIR { + /* Current directory entry */ + struct _wdirent ent; + + /* Private file data */ + WIN32_FIND_DATAW data; + + /* True if data is valid */ + int cached; + + /* Win32 search handle */ + HANDLE handle; + + /* Initial directory name */ + wchar_t *patt; +}; +typedef struct _WDIR _WDIR; + +/* Multi-byte character version */ +struct dirent { + /* Always zero */ + long d_ino; + + /* File position within stream */ + long d_off; + + /* Structure size */ + unsigned short d_reclen; + + /* Length of name without \0 */ + size_t d_namlen; + + /* File type */ + int d_type; + + /* File name */ + char d_name[PATH_MAX+1]; +}; +typedef struct dirent dirent; + +struct DIR { + struct dirent ent; + struct _WDIR *wdirp; +}; +typedef struct DIR DIR; + + +/* Dirent functions */ +static DIR *opendir(const char *dirname); +static _WDIR *_wopendir(const wchar_t *dirname); + +static struct dirent *readdir(DIR *dirp); +static struct _wdirent *_wreaddir(_WDIR *dirp); + +static int readdir_r( + DIR *dirp, struct dirent *entry, struct dirent **result); +static int _wreaddir_r( + _WDIR *dirp, struct _wdirent *entry, struct _wdirent **result); + +static int closedir(DIR *dirp); +static int _wclosedir(_WDIR *dirp); + +static void rewinddir(DIR* dirp); +static void _wrewinddir(_WDIR* dirp); + +static int scandir(const char *dirname, struct dirent ***namelist, + int (*filter)(const struct dirent*), + int (*compare)(const struct dirent**, const struct dirent**)); + +static int alphasort(const struct dirent **a, const struct dirent **b); + +static int versionsort(const struct dirent **a, const struct dirent **b); + +static int strverscmp(const char *a, const char *b); + +/* For compatibility with Symbian */ +#define wdirent _wdirent +#define WDIR _WDIR +#define wopendir _wopendir +#define wreaddir _wreaddir +#define wclosedir _wclosedir +#define wrewinddir _wrewinddir + +/* Compatibility with older Microsoft compilers and non-Microsoft compilers */ +#if !defined(_MSC_VER) || _MSC_VER < 1400 +# define wcstombs_s dirent_wcstombs_s +# define mbstowcs_s dirent_mbstowcs_s +#endif + +/* Optimize dirent_set_errno() away on modern Microsoft compilers */ +#if defined(_MSC_VER) && _MSC_VER >= 1400 +# define dirent_set_errno _set_errno +#endif + + +/* Internal utility functions */ +static WIN32_FIND_DATAW *dirent_first(_WDIR *dirp); +static WIN32_FIND_DATAW *dirent_next(_WDIR *dirp); + +#if !defined(_MSC_VER) || _MSC_VER < 1400 +static int dirent_mbstowcs_s( + size_t *pReturnValue, wchar_t *wcstr, size_t sizeInWords, + const char *mbstr, size_t count); +#endif + +#if !defined(_MSC_VER) || _MSC_VER < 1400 +static int dirent_wcstombs_s( + size_t *pReturnValue, char *mbstr, size_t sizeInBytes, + const wchar_t *wcstr, size_t count); +#endif + +#if !defined(_MSC_VER) || _MSC_VER < 1400 +static void dirent_set_errno(int error); +#endif + + +/* + * Open directory stream DIRNAME for read and return a pointer to the + * internal working area that is used to retrieve individual directory + * entries. + */ +static _WDIR *_wopendir(const wchar_t *dirname) +{ + wchar_t *p; + + /* Must have directory name */ + if (dirname == NULL || dirname[0] == '\0') { + dirent_set_errno(ENOENT); + return NULL; + } + + /* Allocate new _WDIR structure */ + _WDIR *dirp = (_WDIR*) malloc(sizeof(struct _WDIR)); + if (!dirp) + return NULL; + + /* Reset _WDIR structure */ + dirp->handle = INVALID_HANDLE_VALUE; + dirp->patt = NULL; + dirp->cached = 0; + + /* + * Compute the length of full path plus zero terminator + * + * Note that on WinRT there's no way to convert relative paths + * into absolute paths, so just assume it is an absolute path. + */ +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + /* Desktop */ + DWORD n = GetFullPathNameW(dirname, 0, NULL, NULL); +#else + /* WinRT */ + size_t n = wcslen(dirname); +#endif + + /* Allocate room for absolute directory name and search pattern */ + dirp->patt = (wchar_t*) malloc(sizeof(wchar_t) * n + 16); + if (dirp->patt == NULL) + goto exit_closedir; + + /* + * Convert relative directory name to an absolute one. This + * allows rewinddir() to function correctly even when current + * working directory is changed between opendir() and rewinddir(). + * + * Note that on WinRT there's no way to convert relative paths + * into absolute paths, so just assume it is an absolute path. + */ +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + /* Desktop */ + n = GetFullPathNameW(dirname, n, dirp->patt, NULL); + if (n <= 0) + goto exit_closedir; +#else + /* WinRT */ + wcsncpy_s(dirp->patt, n+1, dirname, n); +#endif + + /* Append search pattern \* to the directory name */ + p = dirp->patt + n; + switch (p[-1]) { + case '\\': + case '/': + case ':': + /* Directory ends in path separator, e.g. c:\temp\ */ + /*NOP*/; + break; + + default: + /* Directory name doesn't end in path separator */ + *p++ = '\\'; + } + *p++ = '*'; + *p = '\0'; + + /* Open directory stream and retrieve the first entry */ + if (!dirent_first(dirp)) + goto exit_closedir; + + /* Success */ + return dirp; + + /* Failure */ +exit_closedir: + _wclosedir(dirp); + return NULL; +} + +/* + * Read next directory entry. + * + * Returns pointer to static directory entry which may be overwritten by + * subsequent calls to _wreaddir(). + */ +static struct _wdirent *_wreaddir(_WDIR *dirp) +{ + /* + * Read directory entry to buffer. We can safely ignore the return + * value as entry will be set to NULL in case of error. + */ + struct _wdirent *entry; + (void) _wreaddir_r(dirp, &dirp->ent, &entry); + + /* Return pointer to statically allocated directory entry */ + return entry; +} + +/* + * Read next directory entry. + * + * Returns zero on success. If end of directory stream is reached, then sets + * result to NULL and returns zero. + */ +static int _wreaddir_r( + _WDIR *dirp, struct _wdirent *entry, struct _wdirent **result) +{ + /* Read next directory entry */ + WIN32_FIND_DATAW *datap = dirent_next(dirp); + if (!datap) { + /* Return NULL to indicate end of directory */ + *result = NULL; + return /*OK*/0; + } + + /* + * Copy file name as wide-character string. If the file name is too + * long to fit in to the destination buffer, then truncate file name + * to PATH_MAX characters and zero-terminate the buffer. + */ + size_t n = 0; + while (n < PATH_MAX && datap->cFileName[n] != 0) { + entry->d_name[n] = datap->cFileName[n]; + n++; + } + entry->d_name[n] = 0; + + /* Length of file name excluding zero terminator */ + entry->d_namlen = n; + + /* File type */ + DWORD attr = datap->dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) + entry->d_type = DT_CHR; + else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) + entry->d_type = DT_DIR; + else + entry->d_type = DT_REG; + + /* Reset dummy fields */ + entry->d_ino = 0; + entry->d_off = 0; + entry->d_reclen = sizeof(struct _wdirent); + + /* Set result address */ + *result = entry; + return /*OK*/0; +} + +/* + * Close directory stream opened by opendir() function. This invalidates the + * DIR structure as well as any directory entry read previously by + * _wreaddir(). + */ +static int _wclosedir(_WDIR *dirp) +{ + if (!dirp) { + dirent_set_errno(EBADF); + return /*failure*/-1; + } + + /* Release search handle */ + if (dirp->handle != INVALID_HANDLE_VALUE) + FindClose(dirp->handle); + + /* Release search pattern */ + free(dirp->patt); + + /* Release directory structure */ + free(dirp); + return /*success*/0; +} + +/* + * Rewind directory stream such that _wreaddir() returns the very first + * file name again. + */ +static void _wrewinddir(_WDIR* dirp) +{ + if (!dirp) + return; + + /* Release existing search handle */ + if (dirp->handle != INVALID_HANDLE_VALUE) + FindClose(dirp->handle); + + /* Open new search handle */ + dirent_first(dirp); +} + +/* Get first directory entry */ +static WIN32_FIND_DATAW *dirent_first(_WDIR *dirp) +{ + if (!dirp) + return NULL; + + /* Open directory and retrieve the first entry */ + dirp->handle = FindFirstFileExW( + dirp->patt, FindExInfoStandard, &dirp->data, + FindExSearchNameMatch, NULL, 0); + if (dirp->handle == INVALID_HANDLE_VALUE) + goto error; + + /* A directory entry is now waiting in memory */ + dirp->cached = 1; + return &dirp->data; + +error: + /* Failed to open directory: no directory entry in memory */ + dirp->cached = 0; + + /* Set error code */ + DWORD errorcode = GetLastError(); + switch (errorcode) { + case ERROR_ACCESS_DENIED: + /* No read access to directory */ + dirent_set_errno(EACCES); + break; + + case ERROR_DIRECTORY: + /* Directory name is invalid */ + dirent_set_errno(ENOTDIR); + break; + + case ERROR_PATH_NOT_FOUND: + default: + /* Cannot find the file */ + dirent_set_errno(ENOENT); + } + return NULL; +} + +/* Get next directory entry */ +static WIN32_FIND_DATAW *dirent_next(_WDIR *dirp) +{ + /* Is the next directory entry already in cache? */ + if (dirp->cached) { + /* Yes, a valid directory entry found in memory */ + dirp->cached = 0; + return &dirp->data; + } + + /* No directory entry in cache */ + if (dirp->handle == INVALID_HANDLE_VALUE) + return NULL; + + /* Read the next directory entry from stream */ + if (FindNextFileW(dirp->handle, &dirp->data) == FALSE) + goto exit_close; + + /* Success */ + return &dirp->data; + + /* Failure */ +exit_close: + FindClose(dirp->handle); + dirp->handle = INVALID_HANDLE_VALUE; + return NULL; +} + +/* Open directory stream using plain old C-string */ +static DIR *opendir(const char *dirname) +{ + /* Must have directory name */ + if (dirname == NULL || dirname[0] == '\0') { + dirent_set_errno(ENOENT); + return NULL; + } + + /* Allocate memory for DIR structure */ + struct DIR *dirp = (DIR*) malloc(sizeof(struct DIR)); + if (!dirp) + return NULL; + + /* Convert directory name to wide-character string */ + wchar_t wname[PATH_MAX + 1]; + size_t n; + int error = mbstowcs_s(&n, wname, PATH_MAX + 1, dirname, PATH_MAX+1); + if (error) + goto exit_failure; + + /* Open directory stream using wide-character name */ + dirp->wdirp = _wopendir(wname); + if (!dirp->wdirp) + goto exit_failure; + + /* Success */ + return dirp; + + /* Failure */ +exit_failure: + free(dirp); + return NULL; +} + +/* Read next directory entry */ +static struct dirent *readdir(DIR *dirp) +{ + /* + * Read directory entry to buffer. We can safely ignore the return + * value as entry will be set to NULL in case of error. + */ + struct dirent *entry; + (void) readdir_r(dirp, &dirp->ent, &entry); + + /* Return pointer to statically allocated directory entry */ + return entry; +} + +/* + * Read next directory entry into called-allocated buffer. + * + * Returns zero on success. If the end of directory stream is reached, then + * sets result to NULL and returns zero. + */ +static int readdir_r( + DIR *dirp, struct dirent *entry, struct dirent **result) +{ + /* Read next directory entry */ + WIN32_FIND_DATAW *datap = dirent_next(dirp->wdirp); + if (!datap) { + /* No more directory entries */ + *result = NULL; + return /*OK*/0; + } + + /* Attempt to convert file name to multi-byte string */ + size_t n; + int error = wcstombs_s( + &n, entry->d_name, PATH_MAX + 1, + datap->cFileName, PATH_MAX + 1); + + /* + * If the file name cannot be represented by a multi-byte string, then + * attempt to use old 8+3 file name. This allows the program to + * access files although file names may seem unfamiliar to the user. + * + * Be ware that the code below cannot come up with a short file name + * unless the file system provides one. At least VirtualBox shared + * folders fail to do this. + */ + if (error && datap->cAlternateFileName[0] != '\0') { + error = wcstombs_s( + &n, entry->d_name, PATH_MAX + 1, + datap->cAlternateFileName, PATH_MAX + 1); + } + + if (!error) { + /* Length of file name excluding zero terminator */ + entry->d_namlen = n - 1; + + /* File attributes */ + DWORD attr = datap->dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) + entry->d_type = DT_CHR; + else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) + entry->d_type = DT_DIR; + else + entry->d_type = DT_REG; + + /* Reset dummy fields */ + entry->d_ino = 0; + entry->d_off = 0; + entry->d_reclen = sizeof(struct dirent); + } else { + /* + * Cannot convert file name to multi-byte string so construct + * an erroneous directory entry and return that. Note that + * we cannot return NULL as that would stop the processing + * of directory entries completely. + */ + entry->d_name[0] = '?'; + entry->d_name[1] = '\0'; + entry->d_namlen = 1; + entry->d_type = DT_UNKNOWN; + entry->d_ino = 0; + entry->d_off = -1; + entry->d_reclen = 0; + } + + /* Return pointer to directory entry */ + *result = entry; + return /*OK*/0; +} + +/* Close directory stream */ +static int closedir(DIR *dirp) +{ + int ok; + + if (!dirp) + goto exit_failure; + + /* Close wide-character directory stream */ + ok = _wclosedir(dirp->wdirp); + dirp->wdirp = NULL; + + /* Release multi-byte character version */ + free(dirp); + return ok; + +exit_failure: + /* Invalid directory stream */ + dirent_set_errno(EBADF); + return /*failure*/-1; +} + +/* Rewind directory stream to beginning */ +static void rewinddir(DIR* dirp) +{ + if (!dirp) + return; + + /* Rewind wide-character string directory stream */ + _wrewinddir(dirp->wdirp); +} + +/* Scan directory for entries */ +static int scandir( + const char *dirname, struct dirent ***namelist, + int (*filter)(const struct dirent*), + int (*compare)(const struct dirent**, const struct dirent**)) +{ + int result; + + /* Open directory stream */ + DIR *dir = opendir(dirname); + if (!dir) { + /* Cannot open directory */ + return /*Error*/ -1; + } + + /* Read directory entries to memory */ + struct dirent *tmp = NULL; + struct dirent **files = NULL; + size_t size = 0; + size_t allocated = 0; + while (1) { + /* Allocate room for a temporary directory entry */ + if (!tmp) { + tmp = (struct dirent*) malloc(sizeof(struct dirent)); + if (!tmp) + goto exit_failure; + } + + /* Read directory entry to temporary area */ + struct dirent *entry; + if (readdir_r(dir, tmp, &entry) != /*OK*/0) + goto exit_failure; + + /* Stop if we already read the last directory entry */ + if (entry == NULL) + goto exit_success; + + /* Determine whether to include the entry in results */ + if (filter && !filter(tmp)) + continue; + + /* Enlarge pointer table to make room for another pointer */ + if (size >= allocated) { + /* Compute number of entries in the new table */ + size_t num_entries = size * 2 + 16; + + /* Allocate new pointer table or enlarge existing */ + void *p = realloc(files, sizeof(void*) * num_entries); + if (!p) + goto exit_failure; + + /* Got the memory */ + files = (dirent**) p; + allocated = num_entries; + } + + /* Store the temporary entry to ptr table */ + files[size++] = tmp; + tmp = NULL; + } + +exit_failure: + /* Release allocated file entries */ + for (size_t i = 0; i < size; i++) { + free(files[i]); + } + + /* Release the pointer table */ + free(files); + files = NULL; + + /* Exit with error code */ + result = /*error*/ -1; + goto exit_status; + +exit_success: + /* Sort directory entries */ + qsort(files, size, sizeof(void*), + (int (*) (const void*, const void*)) compare); + + /* Pass pointer table to caller */ + if (namelist) + *namelist = files; + + /* Return the number of directory entries read */ + result = (int) size; + +exit_status: + /* Release temporary directory entry, if we had one */ + free(tmp); + + /* Close directory stream */ + closedir(dir); + return result; +} + +/* Alphabetical sorting */ +static int alphasort(const struct dirent **a, const struct dirent **b) +{ + return strcoll((*a)->d_name, (*b)->d_name); +} + +/* Sort versions */ +static int versionsort(const struct dirent **a, const struct dirent **b) +{ + return strverscmp((*a)->d_name, (*b)->d_name); +} + +/* Compare strings */ +static int strverscmp(const char *a, const char *b) +{ + size_t i = 0; + size_t j; + + /* Find first difference */ + while (a[i] == b[i]) { + if (a[i] == '\0') { + /* No difference */ + return 0; + } + ++i; + } + + /* Count backwards and find the leftmost digit */ + j = i; + while (j > 0 && isdigit(a[j-1])) { + --j; + } + + /* Determine mode of comparison */ + if (a[j] == '0' || b[j] == '0') { + /* Find the next non-zero digit */ + while (a[j] == '0' && a[j] == b[j]) { + j++; + } + + /* String with more digits is smaller, e.g 002 < 01 */ + if (isdigit(a[j])) { + if (!isdigit(b[j])) { + return -1; + } + } else if (isdigit(b[j])) { + return 1; + } + } else if (isdigit(a[j]) && isdigit(b[j])) { + /* Numeric comparison */ + size_t k1 = j; + size_t k2 = j; + + /* Compute number of digits in each string */ + while (isdigit(a[k1])) { + k1++; + } + while (isdigit(b[k2])) { + k2++; + } + + /* Number with more digits is bigger, e.g 999 < 1000 */ + if (k1 < k2) + return -1; + else if (k1 > k2) + return 1; + } + + /* Alphabetical comparison */ + return (int) ((unsigned char) a[i]) - ((unsigned char) b[i]); +} + +/* Convert multi-byte string to wide character string */ +#if !defined(_MSC_VER) || _MSC_VER < 1400 +static int dirent_mbstowcs_s( + size_t *pReturnValue, wchar_t *wcstr, + size_t sizeInWords, const char *mbstr, size_t count) +{ + /* Older Visual Studio or non-Microsoft compiler */ + size_t n = mbstowcs(wcstr, mbstr, sizeInWords); + if (wcstr && n >= count) + return /*error*/ 1; + + /* Zero-terminate output buffer */ + if (wcstr && sizeInWords) { + if (n >= sizeInWords) + n = sizeInWords - 1; + wcstr[n] = 0; + } + + /* Length of multi-byte string with zero terminator */ + if (pReturnValue) { + *pReturnValue = n + 1; + } + + /* Success */ + return 0; +} +#endif + +/* Convert wide-character string to multi-byte string */ +#if !defined(_MSC_VER) || _MSC_VER < 1400 +static int dirent_wcstombs_s( + size_t *pReturnValue, char *mbstr, + size_t sizeInBytes, const wchar_t *wcstr, size_t count) +{ + /* Older Visual Studio or non-Microsoft compiler */ + size_t n = wcstombs(mbstr, wcstr, sizeInBytes); + if (mbstr && n >= count) + return /*error*/1; + + /* Zero-terminate output buffer */ + if (mbstr && sizeInBytes) { + if (n >= sizeInBytes) { + n = sizeInBytes - 1; + } + mbstr[n] = '\0'; + } + + /* Length of resulting multi-bytes string WITH zero-terminator */ + if (pReturnValue) { + *pReturnValue = n + 1; + } + + /* Success */ + return 0; +} +#endif + +/* Set errno variable */ +#if !defined(_MSC_VER) || _MSC_VER < 1400 +static void dirent_set_errno(int error) +{ + /* Non-Microsoft compiler or older Microsoft compiler */ + errno = error; +} +#endif + +#ifdef __cplusplus +} +#endif +#endif /*DIRENT_H*/ + From 57d16f4786688fd2ddaeab9e2023465a79607415 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 16:13:20 +0800 Subject: [PATCH 14/20] Fix typo Signed-off-by: Xuanwo --- build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.rs b/build.rs index f8914d1..0068824 100644 --- a/build.rs +++ b/build.rs @@ -150,7 +150,7 @@ fn main() -> Result<(), Box> { // Sadly, hadoop include `dirent.h` but mvsc doesn't support it. // So we include [tronkko/dirent](https://raw.githubusercontent.com/tronkko/dirent/master/include/dirent.h) instead. if cfg!(feature = "hdfs_3_3") && cfg!(target_os = "windows") { - builder.include("dirent/include") + builder.include("dirent/include"); } builder.compile("hdfs"); From 15b4aa66d7a5accce49b32bbb520af8363826bd3 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 16:35:46 +0800 Subject: [PATCH 15/20] Remove windows test Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 10 +- build.rs | 6 - dirent/include/dirent.h | 1028 -------------------------------------- 3 files changed, 7 insertions(+), 1037 deletions(-) delete mode 100644 dirent/include/dirent.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0eebdc3..1e4309a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-latest, macos-latest, windows-latest ] + # We should support windows, but so far we can't build hadoop on it. + # Visit [Tracking issue of windows support](https://github.com/Xuanwo/hdfs-sys/issues/17) for more details. + os: [ ubuntu-latest, macos-latest ] feature: [ "hdfs_2_2", "hdfs_2_3", @@ -52,7 +54,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ ubuntu-latest, macos-latest, windows-latest ] + # We should support windows, but so far we can't build hadoop on it. + # Visit [Tracking issue of windows support](https://github.com/Xuanwo/hdfs-sys/issues/17) for more details. + os: [ ubuntu-latest, macos-latest ] feature: [ "hdfs_2_2", "hdfs_2_3", @@ -74,7 +78,7 @@ jobs: - uses: actions/setup-java@v3 with: distribution: 'temurin' - java-version: '11' + java-version: '8' - name: Test uses: actions-rs/cargo@v1 with: diff --git a/build.rs b/build.rs index 0068824..25c20c6 100644 --- a/build.rs +++ b/build.rs @@ -147,12 +147,6 @@ fn main() -> Result<(), Box> { builder.file(format!("libhdfs/{version}/jclasses.c")); } - // Sadly, hadoop include `dirent.h` but mvsc doesn't support it. - // So we include [tronkko/dirent](https://raw.githubusercontent.com/tronkko/dirent/master/include/dirent.h) instead. - if cfg!(feature = "hdfs_3_3") && cfg!(target_os = "windows") { - builder.include("dirent/include"); - } - builder.compile("hdfs"); Ok(()) } diff --git a/dirent/include/dirent.h b/dirent/include/dirent.h deleted file mode 100644 index ad52607..0000000 --- a/dirent/include/dirent.h +++ /dev/null @@ -1,1028 +0,0 @@ -/* - * Dirent interface for Microsoft Visual Studio - * - * Copyright (C) 1998-2019 Toni Ronkko - * This file is part of dirent. Dirent may be freely distributed - * under the MIT license. For all details and documentation, see - * https://github.com/tronkko/dirent - */ -#ifndef DIRENT_H -#define DIRENT_H - -/* Hide warnings about unreferenced local functions */ -#if defined(__clang__) -# pragma clang diagnostic ignored "-Wunused-function" -#elif defined(_MSC_VER) -# pragma warning(disable:4505) -#elif defined(__GNUC__) -# pragma GCC diagnostic ignored "-Wunused-function" -#endif - -/* - * Include windows.h without Windows Sockets 1.1 to prevent conflicts with - * Windows Sockets 2.0. - */ -#ifndef WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN -#endif -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Indicates that d_type field is available in dirent structure */ -#define _DIRENT_HAVE_D_TYPE - -/* Indicates that d_namlen field is available in dirent structure */ -#define _DIRENT_HAVE_D_NAMLEN - -/* Entries missing from MSVC 6.0 */ -#if !defined(FILE_ATTRIBUTE_DEVICE) -# define FILE_ATTRIBUTE_DEVICE 0x40 -#endif - -/* File type and permission flags for stat(), general mask */ -#if !defined(S_IFMT) -# define S_IFMT _S_IFMT -#endif - -/* Directory bit */ -#if !defined(S_IFDIR) -# define S_IFDIR _S_IFDIR -#endif - -/* Character device bit */ -#if !defined(S_IFCHR) -# define S_IFCHR _S_IFCHR -#endif - -/* Pipe bit */ -#if !defined(S_IFFIFO) -# define S_IFFIFO _S_IFFIFO -#endif - -/* Regular file bit */ -#if !defined(S_IFREG) -# define S_IFREG _S_IFREG -#endif - -/* Read permission */ -#if !defined(S_IREAD) -# define S_IREAD _S_IREAD -#endif - -/* Write permission */ -#if !defined(S_IWRITE) -# define S_IWRITE _S_IWRITE -#endif - -/* Execute permission */ -#if !defined(S_IEXEC) -# define S_IEXEC _S_IEXEC -#endif - -/* Pipe */ -#if !defined(S_IFIFO) -# define S_IFIFO _S_IFIFO -#endif - -/* Block device */ -#if !defined(S_IFBLK) -# define S_IFBLK 0 -#endif - -/* Link */ -#if !defined(S_IFLNK) -# define S_IFLNK 0 -#endif - -/* Socket */ -#if !defined(S_IFSOCK) -# define S_IFSOCK 0 -#endif - -/* Read user permission */ -#if !defined(S_IRUSR) -# define S_IRUSR S_IREAD -#endif - -/* Write user permission */ -#if !defined(S_IWUSR) -# define S_IWUSR S_IWRITE -#endif - -/* Execute user permission */ -#if !defined(S_IXUSR) -# define S_IXUSR 0 -#endif - -/* Read group permission */ -#if !defined(S_IRGRP) -# define S_IRGRP 0 -#endif - -/* Write group permission */ -#if !defined(S_IWGRP) -# define S_IWGRP 0 -#endif - -/* Execute group permission */ -#if !defined(S_IXGRP) -# define S_IXGRP 0 -#endif - -/* Read others permission */ -#if !defined(S_IROTH) -# define S_IROTH 0 -#endif - -/* Write others permission */ -#if !defined(S_IWOTH) -# define S_IWOTH 0 -#endif - -/* Execute others permission */ -#if !defined(S_IXOTH) -# define S_IXOTH 0 -#endif - -/* Maximum length of file name */ -#if !defined(PATH_MAX) -# define PATH_MAX MAX_PATH -#endif -#if !defined(FILENAME_MAX) -# define FILENAME_MAX MAX_PATH -#endif -#if !defined(NAME_MAX) -# define NAME_MAX FILENAME_MAX -#endif - -/* File type flags for d_type */ -#define DT_UNKNOWN 0 -#define DT_REG S_IFREG -#define DT_DIR S_IFDIR -#define DT_FIFO S_IFIFO -#define DT_SOCK S_IFSOCK -#define DT_CHR S_IFCHR -#define DT_BLK S_IFBLK -#define DT_LNK S_IFLNK - -/* Macros for converting between st_mode and d_type */ -#define IFTODT(mode) ((mode) & S_IFMT) -#define DTTOIF(type) (type) - -/* - * File type macros. Note that block devices, sockets and links cannot be - * distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are - * only defined for compatibility. These macros should always return false - * on Windows. - */ -#if !defined(S_ISFIFO) -# define S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFIFO) -#endif -#if !defined(S_ISDIR) -# define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) -#endif -#if !defined(S_ISREG) -# define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) -#endif -#if !defined(S_ISLNK) -# define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) -#endif -#if !defined(S_ISSOCK) -# define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) -#endif -#if !defined(S_ISCHR) -# define S_ISCHR(mode) (((mode) & S_IFMT) == S_IFCHR) -#endif -#if !defined(S_ISBLK) -# define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) -#endif - -/* Return the exact length of the file name without zero terminator */ -#define _D_EXACT_NAMLEN(p) ((p)->d_namlen) - -/* Return the maximum size of a file name */ -#define _D_ALLOC_NAMLEN(p) ((PATH_MAX)+1) - - -#ifdef __cplusplus -extern "C" { -#endif - - -/* Wide-character version */ -struct _wdirent { - /* Always zero */ - long d_ino; - - /* File position within stream */ - long d_off; - - /* Structure size */ - unsigned short d_reclen; - - /* Length of name without \0 */ - size_t d_namlen; - - /* File type */ - int d_type; - - /* File name */ - wchar_t d_name[PATH_MAX+1]; -}; -typedef struct _wdirent _wdirent; - -struct _WDIR { - /* Current directory entry */ - struct _wdirent ent; - - /* Private file data */ - WIN32_FIND_DATAW data; - - /* True if data is valid */ - int cached; - - /* Win32 search handle */ - HANDLE handle; - - /* Initial directory name */ - wchar_t *patt; -}; -typedef struct _WDIR _WDIR; - -/* Multi-byte character version */ -struct dirent { - /* Always zero */ - long d_ino; - - /* File position within stream */ - long d_off; - - /* Structure size */ - unsigned short d_reclen; - - /* Length of name without \0 */ - size_t d_namlen; - - /* File type */ - int d_type; - - /* File name */ - char d_name[PATH_MAX+1]; -}; -typedef struct dirent dirent; - -struct DIR { - struct dirent ent; - struct _WDIR *wdirp; -}; -typedef struct DIR DIR; - - -/* Dirent functions */ -static DIR *opendir(const char *dirname); -static _WDIR *_wopendir(const wchar_t *dirname); - -static struct dirent *readdir(DIR *dirp); -static struct _wdirent *_wreaddir(_WDIR *dirp); - -static int readdir_r( - DIR *dirp, struct dirent *entry, struct dirent **result); -static int _wreaddir_r( - _WDIR *dirp, struct _wdirent *entry, struct _wdirent **result); - -static int closedir(DIR *dirp); -static int _wclosedir(_WDIR *dirp); - -static void rewinddir(DIR* dirp); -static void _wrewinddir(_WDIR* dirp); - -static int scandir(const char *dirname, struct dirent ***namelist, - int (*filter)(const struct dirent*), - int (*compare)(const struct dirent**, const struct dirent**)); - -static int alphasort(const struct dirent **a, const struct dirent **b); - -static int versionsort(const struct dirent **a, const struct dirent **b); - -static int strverscmp(const char *a, const char *b); - -/* For compatibility with Symbian */ -#define wdirent _wdirent -#define WDIR _WDIR -#define wopendir _wopendir -#define wreaddir _wreaddir -#define wclosedir _wclosedir -#define wrewinddir _wrewinddir - -/* Compatibility with older Microsoft compilers and non-Microsoft compilers */ -#if !defined(_MSC_VER) || _MSC_VER < 1400 -# define wcstombs_s dirent_wcstombs_s -# define mbstowcs_s dirent_mbstowcs_s -#endif - -/* Optimize dirent_set_errno() away on modern Microsoft compilers */ -#if defined(_MSC_VER) && _MSC_VER >= 1400 -# define dirent_set_errno _set_errno -#endif - - -/* Internal utility functions */ -static WIN32_FIND_DATAW *dirent_first(_WDIR *dirp); -static WIN32_FIND_DATAW *dirent_next(_WDIR *dirp); - -#if !defined(_MSC_VER) || _MSC_VER < 1400 -static int dirent_mbstowcs_s( - size_t *pReturnValue, wchar_t *wcstr, size_t sizeInWords, - const char *mbstr, size_t count); -#endif - -#if !defined(_MSC_VER) || _MSC_VER < 1400 -static int dirent_wcstombs_s( - size_t *pReturnValue, char *mbstr, size_t sizeInBytes, - const wchar_t *wcstr, size_t count); -#endif - -#if !defined(_MSC_VER) || _MSC_VER < 1400 -static void dirent_set_errno(int error); -#endif - - -/* - * Open directory stream DIRNAME for read and return a pointer to the - * internal working area that is used to retrieve individual directory - * entries. - */ -static _WDIR *_wopendir(const wchar_t *dirname) -{ - wchar_t *p; - - /* Must have directory name */ - if (dirname == NULL || dirname[0] == '\0') { - dirent_set_errno(ENOENT); - return NULL; - } - - /* Allocate new _WDIR structure */ - _WDIR *dirp = (_WDIR*) malloc(sizeof(struct _WDIR)); - if (!dirp) - return NULL; - - /* Reset _WDIR structure */ - dirp->handle = INVALID_HANDLE_VALUE; - dirp->patt = NULL; - dirp->cached = 0; - - /* - * Compute the length of full path plus zero terminator - * - * Note that on WinRT there's no way to convert relative paths - * into absolute paths, so just assume it is an absolute path. - */ -#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) - /* Desktop */ - DWORD n = GetFullPathNameW(dirname, 0, NULL, NULL); -#else - /* WinRT */ - size_t n = wcslen(dirname); -#endif - - /* Allocate room for absolute directory name and search pattern */ - dirp->patt = (wchar_t*) malloc(sizeof(wchar_t) * n + 16); - if (dirp->patt == NULL) - goto exit_closedir; - - /* - * Convert relative directory name to an absolute one. This - * allows rewinddir() to function correctly even when current - * working directory is changed between opendir() and rewinddir(). - * - * Note that on WinRT there's no way to convert relative paths - * into absolute paths, so just assume it is an absolute path. - */ -#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) - /* Desktop */ - n = GetFullPathNameW(dirname, n, dirp->patt, NULL); - if (n <= 0) - goto exit_closedir; -#else - /* WinRT */ - wcsncpy_s(dirp->patt, n+1, dirname, n); -#endif - - /* Append search pattern \* to the directory name */ - p = dirp->patt + n; - switch (p[-1]) { - case '\\': - case '/': - case ':': - /* Directory ends in path separator, e.g. c:\temp\ */ - /*NOP*/; - break; - - default: - /* Directory name doesn't end in path separator */ - *p++ = '\\'; - } - *p++ = '*'; - *p = '\0'; - - /* Open directory stream and retrieve the first entry */ - if (!dirent_first(dirp)) - goto exit_closedir; - - /* Success */ - return dirp; - - /* Failure */ -exit_closedir: - _wclosedir(dirp); - return NULL; -} - -/* - * Read next directory entry. - * - * Returns pointer to static directory entry which may be overwritten by - * subsequent calls to _wreaddir(). - */ -static struct _wdirent *_wreaddir(_WDIR *dirp) -{ - /* - * Read directory entry to buffer. We can safely ignore the return - * value as entry will be set to NULL in case of error. - */ - struct _wdirent *entry; - (void) _wreaddir_r(dirp, &dirp->ent, &entry); - - /* Return pointer to statically allocated directory entry */ - return entry; -} - -/* - * Read next directory entry. - * - * Returns zero on success. If end of directory stream is reached, then sets - * result to NULL and returns zero. - */ -static int _wreaddir_r( - _WDIR *dirp, struct _wdirent *entry, struct _wdirent **result) -{ - /* Read next directory entry */ - WIN32_FIND_DATAW *datap = dirent_next(dirp); - if (!datap) { - /* Return NULL to indicate end of directory */ - *result = NULL; - return /*OK*/0; - } - - /* - * Copy file name as wide-character string. If the file name is too - * long to fit in to the destination buffer, then truncate file name - * to PATH_MAX characters and zero-terminate the buffer. - */ - size_t n = 0; - while (n < PATH_MAX && datap->cFileName[n] != 0) { - entry->d_name[n] = datap->cFileName[n]; - n++; - } - entry->d_name[n] = 0; - - /* Length of file name excluding zero terminator */ - entry->d_namlen = n; - - /* File type */ - DWORD attr = datap->dwFileAttributes; - if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) - entry->d_type = DT_CHR; - else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) - entry->d_type = DT_DIR; - else - entry->d_type = DT_REG; - - /* Reset dummy fields */ - entry->d_ino = 0; - entry->d_off = 0; - entry->d_reclen = sizeof(struct _wdirent); - - /* Set result address */ - *result = entry; - return /*OK*/0; -} - -/* - * Close directory stream opened by opendir() function. This invalidates the - * DIR structure as well as any directory entry read previously by - * _wreaddir(). - */ -static int _wclosedir(_WDIR *dirp) -{ - if (!dirp) { - dirent_set_errno(EBADF); - return /*failure*/-1; - } - - /* Release search handle */ - if (dirp->handle != INVALID_HANDLE_VALUE) - FindClose(dirp->handle); - - /* Release search pattern */ - free(dirp->patt); - - /* Release directory structure */ - free(dirp); - return /*success*/0; -} - -/* - * Rewind directory stream such that _wreaddir() returns the very first - * file name again. - */ -static void _wrewinddir(_WDIR* dirp) -{ - if (!dirp) - return; - - /* Release existing search handle */ - if (dirp->handle != INVALID_HANDLE_VALUE) - FindClose(dirp->handle); - - /* Open new search handle */ - dirent_first(dirp); -} - -/* Get first directory entry */ -static WIN32_FIND_DATAW *dirent_first(_WDIR *dirp) -{ - if (!dirp) - return NULL; - - /* Open directory and retrieve the first entry */ - dirp->handle = FindFirstFileExW( - dirp->patt, FindExInfoStandard, &dirp->data, - FindExSearchNameMatch, NULL, 0); - if (dirp->handle == INVALID_HANDLE_VALUE) - goto error; - - /* A directory entry is now waiting in memory */ - dirp->cached = 1; - return &dirp->data; - -error: - /* Failed to open directory: no directory entry in memory */ - dirp->cached = 0; - - /* Set error code */ - DWORD errorcode = GetLastError(); - switch (errorcode) { - case ERROR_ACCESS_DENIED: - /* No read access to directory */ - dirent_set_errno(EACCES); - break; - - case ERROR_DIRECTORY: - /* Directory name is invalid */ - dirent_set_errno(ENOTDIR); - break; - - case ERROR_PATH_NOT_FOUND: - default: - /* Cannot find the file */ - dirent_set_errno(ENOENT); - } - return NULL; -} - -/* Get next directory entry */ -static WIN32_FIND_DATAW *dirent_next(_WDIR *dirp) -{ - /* Is the next directory entry already in cache? */ - if (dirp->cached) { - /* Yes, a valid directory entry found in memory */ - dirp->cached = 0; - return &dirp->data; - } - - /* No directory entry in cache */ - if (dirp->handle == INVALID_HANDLE_VALUE) - return NULL; - - /* Read the next directory entry from stream */ - if (FindNextFileW(dirp->handle, &dirp->data) == FALSE) - goto exit_close; - - /* Success */ - return &dirp->data; - - /* Failure */ -exit_close: - FindClose(dirp->handle); - dirp->handle = INVALID_HANDLE_VALUE; - return NULL; -} - -/* Open directory stream using plain old C-string */ -static DIR *opendir(const char *dirname) -{ - /* Must have directory name */ - if (dirname == NULL || dirname[0] == '\0') { - dirent_set_errno(ENOENT); - return NULL; - } - - /* Allocate memory for DIR structure */ - struct DIR *dirp = (DIR*) malloc(sizeof(struct DIR)); - if (!dirp) - return NULL; - - /* Convert directory name to wide-character string */ - wchar_t wname[PATH_MAX + 1]; - size_t n; - int error = mbstowcs_s(&n, wname, PATH_MAX + 1, dirname, PATH_MAX+1); - if (error) - goto exit_failure; - - /* Open directory stream using wide-character name */ - dirp->wdirp = _wopendir(wname); - if (!dirp->wdirp) - goto exit_failure; - - /* Success */ - return dirp; - - /* Failure */ -exit_failure: - free(dirp); - return NULL; -} - -/* Read next directory entry */ -static struct dirent *readdir(DIR *dirp) -{ - /* - * Read directory entry to buffer. We can safely ignore the return - * value as entry will be set to NULL in case of error. - */ - struct dirent *entry; - (void) readdir_r(dirp, &dirp->ent, &entry); - - /* Return pointer to statically allocated directory entry */ - return entry; -} - -/* - * Read next directory entry into called-allocated buffer. - * - * Returns zero on success. If the end of directory stream is reached, then - * sets result to NULL and returns zero. - */ -static int readdir_r( - DIR *dirp, struct dirent *entry, struct dirent **result) -{ - /* Read next directory entry */ - WIN32_FIND_DATAW *datap = dirent_next(dirp->wdirp); - if (!datap) { - /* No more directory entries */ - *result = NULL; - return /*OK*/0; - } - - /* Attempt to convert file name to multi-byte string */ - size_t n; - int error = wcstombs_s( - &n, entry->d_name, PATH_MAX + 1, - datap->cFileName, PATH_MAX + 1); - - /* - * If the file name cannot be represented by a multi-byte string, then - * attempt to use old 8+3 file name. This allows the program to - * access files although file names may seem unfamiliar to the user. - * - * Be ware that the code below cannot come up with a short file name - * unless the file system provides one. At least VirtualBox shared - * folders fail to do this. - */ - if (error && datap->cAlternateFileName[0] != '\0') { - error = wcstombs_s( - &n, entry->d_name, PATH_MAX + 1, - datap->cAlternateFileName, PATH_MAX + 1); - } - - if (!error) { - /* Length of file name excluding zero terminator */ - entry->d_namlen = n - 1; - - /* File attributes */ - DWORD attr = datap->dwFileAttributes; - if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) - entry->d_type = DT_CHR; - else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) - entry->d_type = DT_DIR; - else - entry->d_type = DT_REG; - - /* Reset dummy fields */ - entry->d_ino = 0; - entry->d_off = 0; - entry->d_reclen = sizeof(struct dirent); - } else { - /* - * Cannot convert file name to multi-byte string so construct - * an erroneous directory entry and return that. Note that - * we cannot return NULL as that would stop the processing - * of directory entries completely. - */ - entry->d_name[0] = '?'; - entry->d_name[1] = '\0'; - entry->d_namlen = 1; - entry->d_type = DT_UNKNOWN; - entry->d_ino = 0; - entry->d_off = -1; - entry->d_reclen = 0; - } - - /* Return pointer to directory entry */ - *result = entry; - return /*OK*/0; -} - -/* Close directory stream */ -static int closedir(DIR *dirp) -{ - int ok; - - if (!dirp) - goto exit_failure; - - /* Close wide-character directory stream */ - ok = _wclosedir(dirp->wdirp); - dirp->wdirp = NULL; - - /* Release multi-byte character version */ - free(dirp); - return ok; - -exit_failure: - /* Invalid directory stream */ - dirent_set_errno(EBADF); - return /*failure*/-1; -} - -/* Rewind directory stream to beginning */ -static void rewinddir(DIR* dirp) -{ - if (!dirp) - return; - - /* Rewind wide-character string directory stream */ - _wrewinddir(dirp->wdirp); -} - -/* Scan directory for entries */ -static int scandir( - const char *dirname, struct dirent ***namelist, - int (*filter)(const struct dirent*), - int (*compare)(const struct dirent**, const struct dirent**)) -{ - int result; - - /* Open directory stream */ - DIR *dir = opendir(dirname); - if (!dir) { - /* Cannot open directory */ - return /*Error*/ -1; - } - - /* Read directory entries to memory */ - struct dirent *tmp = NULL; - struct dirent **files = NULL; - size_t size = 0; - size_t allocated = 0; - while (1) { - /* Allocate room for a temporary directory entry */ - if (!tmp) { - tmp = (struct dirent*) malloc(sizeof(struct dirent)); - if (!tmp) - goto exit_failure; - } - - /* Read directory entry to temporary area */ - struct dirent *entry; - if (readdir_r(dir, tmp, &entry) != /*OK*/0) - goto exit_failure; - - /* Stop if we already read the last directory entry */ - if (entry == NULL) - goto exit_success; - - /* Determine whether to include the entry in results */ - if (filter && !filter(tmp)) - continue; - - /* Enlarge pointer table to make room for another pointer */ - if (size >= allocated) { - /* Compute number of entries in the new table */ - size_t num_entries = size * 2 + 16; - - /* Allocate new pointer table or enlarge existing */ - void *p = realloc(files, sizeof(void*) * num_entries); - if (!p) - goto exit_failure; - - /* Got the memory */ - files = (dirent**) p; - allocated = num_entries; - } - - /* Store the temporary entry to ptr table */ - files[size++] = tmp; - tmp = NULL; - } - -exit_failure: - /* Release allocated file entries */ - for (size_t i = 0; i < size; i++) { - free(files[i]); - } - - /* Release the pointer table */ - free(files); - files = NULL; - - /* Exit with error code */ - result = /*error*/ -1; - goto exit_status; - -exit_success: - /* Sort directory entries */ - qsort(files, size, sizeof(void*), - (int (*) (const void*, const void*)) compare); - - /* Pass pointer table to caller */ - if (namelist) - *namelist = files; - - /* Return the number of directory entries read */ - result = (int) size; - -exit_status: - /* Release temporary directory entry, if we had one */ - free(tmp); - - /* Close directory stream */ - closedir(dir); - return result; -} - -/* Alphabetical sorting */ -static int alphasort(const struct dirent **a, const struct dirent **b) -{ - return strcoll((*a)->d_name, (*b)->d_name); -} - -/* Sort versions */ -static int versionsort(const struct dirent **a, const struct dirent **b) -{ - return strverscmp((*a)->d_name, (*b)->d_name); -} - -/* Compare strings */ -static int strverscmp(const char *a, const char *b) -{ - size_t i = 0; - size_t j; - - /* Find first difference */ - while (a[i] == b[i]) { - if (a[i] == '\0') { - /* No difference */ - return 0; - } - ++i; - } - - /* Count backwards and find the leftmost digit */ - j = i; - while (j > 0 && isdigit(a[j-1])) { - --j; - } - - /* Determine mode of comparison */ - if (a[j] == '0' || b[j] == '0') { - /* Find the next non-zero digit */ - while (a[j] == '0' && a[j] == b[j]) { - j++; - } - - /* String with more digits is smaller, e.g 002 < 01 */ - if (isdigit(a[j])) { - if (!isdigit(b[j])) { - return -1; - } - } else if (isdigit(b[j])) { - return 1; - } - } else if (isdigit(a[j]) && isdigit(b[j])) { - /* Numeric comparison */ - size_t k1 = j; - size_t k2 = j; - - /* Compute number of digits in each string */ - while (isdigit(a[k1])) { - k1++; - } - while (isdigit(b[k2])) { - k2++; - } - - /* Number with more digits is bigger, e.g 999 < 1000 */ - if (k1 < k2) - return -1; - else if (k1 > k2) - return 1; - } - - /* Alphabetical comparison */ - return (int) ((unsigned char) a[i]) - ((unsigned char) b[i]); -} - -/* Convert multi-byte string to wide character string */ -#if !defined(_MSC_VER) || _MSC_VER < 1400 -static int dirent_mbstowcs_s( - size_t *pReturnValue, wchar_t *wcstr, - size_t sizeInWords, const char *mbstr, size_t count) -{ - /* Older Visual Studio or non-Microsoft compiler */ - size_t n = mbstowcs(wcstr, mbstr, sizeInWords); - if (wcstr && n >= count) - return /*error*/ 1; - - /* Zero-terminate output buffer */ - if (wcstr && sizeInWords) { - if (n >= sizeInWords) - n = sizeInWords - 1; - wcstr[n] = 0; - } - - /* Length of multi-byte string with zero terminator */ - if (pReturnValue) { - *pReturnValue = n + 1; - } - - /* Success */ - return 0; -} -#endif - -/* Convert wide-character string to multi-byte string */ -#if !defined(_MSC_VER) || _MSC_VER < 1400 -static int dirent_wcstombs_s( - size_t *pReturnValue, char *mbstr, - size_t sizeInBytes, const wchar_t *wcstr, size_t count) -{ - /* Older Visual Studio or non-Microsoft compiler */ - size_t n = wcstombs(mbstr, wcstr, sizeInBytes); - if (mbstr && n >= count) - return /*error*/1; - - /* Zero-terminate output buffer */ - if (mbstr && sizeInBytes) { - if (n >= sizeInBytes) { - n = sizeInBytes - 1; - } - mbstr[n] = '\0'; - } - - /* Length of resulting multi-bytes string WITH zero-terminator */ - if (pReturnValue) { - *pReturnValue = n + 1; - } - - /* Success */ - return 0; -} -#endif - -/* Set errno variable */ -#if !defined(_MSC_VER) || _MSC_VER < 1400 -static void dirent_set_errno(int error) -{ - /* Non-Microsoft compiler or older Microsoft compiler */ - errno = error; -} -#endif - -#ifdef __cplusplus -} -#endif -#endif /*DIRENT_H*/ - From 652672a4fe53f226030a3e86dd400b83ae08f4a7 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 16:49:43 +0800 Subject: [PATCH 16/20] Search more place Signed-off-by: Xuanwo --- build.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.rs b/build.rs index 25c20c6..b15ed20 100644 --- a/build.rs +++ b/build.rs @@ -8,8 +8,15 @@ fn main() -> Result<(), Box> { // Make sure jvm has been linked. let java_home = env::var("JAVA_HOME")?; + + // I don't know where to find libjvm.so exactly. + // The only thing I can do is search everywhere I know. + println!("cargo:rustc-link-search=native={java_home}/bin/client"); + println!("cargo:rustc-link-search=native={java_home}/bin/server"); println!("cargo:rustc-link-search=native={java_home}/lib/server"); println!("cargo:rustc-link-search=native={java_home}/lib/amd64/server"); + println!("cargo:rustc-link-search=native={java_home}/jre/lib/server"); + println!("cargo:rustc-link-search=native={java_home}/jre/lib/amd64/server"); println!("cargo:rustc-link-lib=jvm"); // Static link compiled `libhdfs.a` From ee9e10e71e3b4828cb44abda284cd5c95afbf344 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 16:58:01 +0800 Subject: [PATCH 17/20] Fix lib not found Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 1 + build.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e4309a..2b0daba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,3 +87,4 @@ jobs: env: RUST_LOG: DEBUG RUST_BACKTRACE: full + LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }} diff --git a/build.rs b/build.rs index b15ed20..8fe5ef5 100644 --- a/build.rs +++ b/build.rs @@ -7,7 +7,7 @@ fn main() -> Result<(), Box> { } // Make sure jvm has been linked. - let java_home = env::var("JAVA_HOME")?; + let java_home = env::var("JAVA_HOME").expect("JAVA_HOME must be set"); // I don't know where to find libjvm.so exactly. // The only thing I can do is search everywhere I know. From 390e832960ffba91513be5b82b0a01136f2764cb Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 17:05:49 +0800 Subject: [PATCH 18/20] Fix jvm not found Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b0daba..020febe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,4 +87,4 @@ jobs: env: RUST_LOG: DEBUG RUST_BACKTRACE: full - LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }} + LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.JAVA_HOME }}/lib/amd64/server From bf7c75074ac1a0a8dd7713bce9d0a78b16242f88 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 17:07:31 +0800 Subject: [PATCH 19/20] Fix not lib not found Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 020febe..d12d22f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,4 +87,4 @@ jobs: env: RUST_LOG: DEBUG RUST_BACKTRACE: full - LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.JAVA_HOME }}/lib/amd64/server + LD_LIBRARY_PATH: ${{ env.JAVA_HOME }}/lib/server:${{ env.JAVA_HOME }}/lib/amd64/server:${{ env.JAVA_HOME }}/jre/lib/server:${{ env.JAVA_HOME }}/jre/lib/amd64/server From f1de148e2777323c08500ae472688802a707f7d2 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 May 2022 17:16:55 +0800 Subject: [PATCH 20/20] Remove extra build steps Signed-off-by: Xuanwo --- .github/workflows/ci.yml | 36 +++++------------------------------- README.md | 2 +- 2 files changed, 6 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d12d22f..4811e8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,37 +19,6 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} - build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - # We should support windows, but so far we can't build hadoop on it. - # Visit [Tracking issue of windows support](https://github.com/Xuanwo/hdfs-sys/issues/17) for more details. - os: [ ubuntu-latest, macos-latest ] - feature: [ - "hdfs_2_2", - "hdfs_2_3", - "hdfs_2_4", - "hdfs_2_5", - "hdfs_2_6", - "hdfs_2_7", - "hdfs_2_8", - "hdfs_2_9", - "hdfs_2_10", - "hdfs_3_0", - "hdfs_3_1", - "hdfs_3_2", - "hdfs_3_3", - ] - - steps: - - uses: actions/checkout@v3 - - name: Build - uses: actions-rs/cargo@v1 - with: - command: build - args: --features ${{ matrix.feature }} - unit: runs-on: ${{ matrix.os }} strategy: @@ -79,6 +48,11 @@ jobs: with: distribution: 'temurin' java-version: '8' + - name: Build + uses: actions-rs/cargo@v1 + with: + command: build + args: --features ${{ matrix.feature }} - name: Test uses: actions-rs/cargo@v1 with: diff --git a/README.md b/README.md index 415738b..2fd03c6 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ NOTE: `hdfs-sys` will ignore linking if `DOCS_RS` is set to build docs. ## Runtime -`libhdfs` uses JNI to call functions provided by jars that provided by hadoop releases. Please make sure `CLASSPATH` is set correctly: +`hdfs-sys` uses JNI to call functions provided by jars that provided by hadoop releases. Please make sure `CLASSPATH` is set correctly before calling any functions provided by `hdfs-sys`: ```shell export JAVA_HOME=/path/to/java