Merge pull request #98 from paulsengroup/feature/rework-pyarrow

Use the Arrow C and PyCapsule data interfaces to share data with Python

pyproject.toml

@@ -6,11 +6,8 @@
 requires = [
   "conan>=2.0.5",
   "nanobind>=2", # This is required in order to run stubgen
-  "numpy",
-  "pandas>=2.1,!=2.2.0",
-  "pyarrow==17.0",
+  "numpy>=2",
   "scikit-build-core>=0.10",
-  "scipy",
   "typing_extensions",
 ]
 
@@ -31,21 +28,27 @@ classifiers = [
   "License :: OSI Approved :: MIT License"
 ]
 
-dependencies = [
+dependencies = []
+
+optional-dependencies.numpy = [
   "numpy",
-  "pyarrow==17.0",
 ]
 
 optional-dependencies.pandas = [
+  "hictkpy[pyarrow]",
   "pandas>=2.1.0,!=2.2.0",
 ]
 
+optional-dependencies.pyarrow = [
+  "pyarrow>=16",
+]
+
 optional-dependencies.scipy = [
   "scipy",
 ]
 
 optional-dependencies.all = [
-  "hictkpy[pandas,scipy]",
+  "hictkpy[numpy,pandas,pyarrow,scipy]",
 ]
 
 optional-dependencies.test = [

src/CMakeLists.txt

@@ -2,10 +2,18 @@
 #
 # SPDX-License-Identifier: MIT
 
+find_package(
+  Python
+  3.9
+  COMPONENTS
+    Interpreter
+    NumPy
+  REQUIRED
+)
+
 find_package(Arrow REQUIRED)
 find_package(FMT REQUIRED)
 find_package(nanobind REQUIRED)
-find_package(Pyarrow REQUIRED)
 find_package(spdlog REQUIRED)
 find_package(Filesystem REQUIRED)
 
@@ -25,16 +33,15 @@ nanobind_add_module(
         "${CMAKE_CURRENT_SOURCE_DIR}/pixel_selector.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/reference.cpp"
         "${CMAKE_CURRENT_SOURCE_DIR}/singlecell_file.cpp"
+        "${CMAKE_CURRENT_SOURCE_DIR}/to_pyarrow.cpp"
 )
 
 target_include_directories(
   _hictkpy
   PRIVATE
     include
-    "${PYARROW_INCLUDE_DIR}"
-    "${NUMPY_INCLUDE_DIR}"
+    "${Python_NumPy_INCLUDE_DIR}"
 )
-target_link_directories(_hictkpy PRIVATE "${PYARROW_LIB_DIRS}")
 
 target_link_libraries(
   _hictkpy
@@ -45,7 +52,7 @@ target_link_libraries(
     hictk::file
     hictk::hic
     hictk::transformers
-    Arrow::python
+    Arrow::arrow_$<IF:$<BOOL:${BUILD_SHARED_LIBS}>,shared,static>
     fmt::fmt-header-only
     spdlog::spdlog_header_only
     std::filesystem

src/hictkpy.cpp

@@ -2,8 +2,6 @@
 //
 // SPDX-License-Identifier: MIT
 
-#include <arrow/config.h>
-#include <arrow/python/pyarrow.h>
 #include <spdlog/spdlog.h>
 
 #include <cstdint>
@@ -34,12 +32,6 @@ namespace hictkpy {
 NB_MODULE(_hictkpy, m) {
   [[maybe_unused]] const auto logger = init_logger();
 
-  if (arrow::py::import_pyarrow() == -1) {
-    throw std::runtime_error("failed to initialize pyarrow runtime");
-  }
-
-  m.attr("__hictkpy_arrow_version__") =
-      std::make_tuple(ARROW_VERSION_MAJOR, ARROW_VERSION_MINOR, ARROW_VERSION_PATCH);
   m.attr("__hictk_version__") = hictk::config::version::str();
 
   m.doc() = "Blazing fast toolkit to work with .hic and .cool files.";

src/hictkpy/__init__.py

@@ -3,27 +3,11 @@
 # SPDX-License-Identifier: MIT
 
 
-def _load_pyarrow_and_check_abi_compat():
-    import pyarrow as pa
+def _get_hictkpy_version() -> str:
+    from importlib.metadata import version
 
-    from ._hictkpy import __hictkpy_arrow_version__
+    return version("hictkpy")
 
-    major, minor, patch = __hictkpy_arrow_version__
-
-    if not pa.__version__.startswith(f"{major}.{minor}"):
-        raise ImportError(
-            "Detected Arrow ABI version mismatch!\n"
-            f"hictkpy was compiled with Arrow v{major}.{minor}.{patch}, which is not ABI compatible with the currently "
-            f"installed version of pyarrow (v{pa.__version__}).\n"
-            'Please install a compatible version of pyarrow with e.g. "pip install '
-            f'pyarrow=={major}.{minor}".'
-        )
-
-
-_load_pyarrow_and_check_abi_compat()
-
-
-from importlib.metadata import version
 
 from ._hictkpy import (
     File,
@@ -39,7 +23,7 @@ def _load_pyarrow_and_check_abi_compat():
     is_scool_file,
 )
 
-__version__ = version("hictkpy")
+__version__ = _get_hictkpy_version()
 __all__ = [
     "__doc__",
     "File",