From 1bad63701c601b8c61ef95a8bb91c20f39450cf7 Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 16 Jun 2024 20:48:14 -0500 Subject: [PATCH] =?UTF-8?q?Bump=20version:=200.3.1=20=E2=86=92=200.4.0=20(?= =?UTF-8?q?#29)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Matt --- .bumpversion.cfg | 2 +- .github/workflows/build_and_upload_conda.yaml | 9 +- conda/conda_build_config.yaml | 14 +- conda/lintdb/build-lib.sh | 10 +- conda/lintdb/build-pkg.sh | 4 +- conda/lintdb/meta.yaml | 38 +++--- lintdb/python/CMakeLists.txt | 4 +- lintdb/version.h | 2 +- ports/rocksdb/0001-fix-dependencies.patch | 122 ++++++++++++++++++ ports/rocksdb/portfile.cmake | 68 ++++++++++ ports/rocksdb/vcpkg.json | 73 +++++++++++ tests/CMakeLists.txt | 8 +- vcpkg.json | 2 +- version.txt | 2 +- 14 files changed, 316 insertions(+), 42 deletions(-) create mode 100644 ports/rocksdb/0001-fix-dependencies.patch create mode 100644 ports/rocksdb/portfile.cmake create mode 100644 ports/rocksdb/vcpkg.json diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c3f4d3e..8d11e2b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.1 +current_version = 0.4.0 commit = True tag = True diff --git a/.github/workflows/build_and_upload_conda.yaml b/.github/workflows/build_and_upload_conda.yaml index 5182ff5..ae3fa64 100644 --- a/.github/workflows/build_and_upload_conda.yaml +++ b/.github/workflows/build_and_upload_conda.yaml @@ -63,7 +63,7 @@ jobs: - name: Update environment run: - mamba env update -n lintdb-build -f + conda env update -n lintdb-build -f conda/environment.yaml if: steps.cache.outputs.cache-hit != 'true' @@ -81,16 +81,17 @@ jobs: MKLROOT: ${{ github.workspace }}/builds/release/vcpkg_installed/x64-linux/lib/intel64 run: | label="" + upload="yes" if [ "${{github.event.action}}" == "released" ]; then label="main" elif [ "${{github.event.action}}" == "prereleased" ]; then label="dev" else - echo "The trigger type ${{github.event.action}} is not compatible with `label: auto`" - exit 1 + label="nightly" + upload="no" fi conda activate lintdb-build - conda config --set anaconda_upload yes + conda config --set anaconda_upload ${upload} out_dir=`mktemp -d -t compilation-XXXXXXXXXX` echo "::group::Building conda package for host platform" echo "conda build lintdb --user deployql --cache-dir "conda-${{ runner.os }}--${{ runner.arch }}-build-cache" --output-folder $out_dir" diff --git a/conda/conda_build_config.yaml b/conda/conda_build_config.yaml index 3865090..ff4d201 100644 --- a/conda/conda_build_config.yaml +++ b/conda/conda_build_config.yaml @@ -1,9 +1,17 @@ python: - - 3.9 - - 3.10 +# - 3.9 +# - 3.10 - 3.11 - - 3.12 +# - 3.12 MACOSX_SDK_VERSION: # [osx and x86_64] - "10.13" # [osx and x86_64] MACOSX_DEPLOYMENT_TARGET: # [osx and x86_64] - "10.13" # [osx and x86_64] +c_compiler: + - clang +c_compiler_version: + - 18 +cxx_compiler: + - clangxx +cxx_compiler_version: + - 18 \ No newline at end of file diff --git a/conda/lintdb/build-lib.sh b/conda/lintdb/build-lib.sh index c5821e8..a009dee 100644 --- a/conda/lintdb/build-lib.sh +++ b/conda/lintdb/build-lib.sh @@ -2,15 +2,19 @@ set -e -cmake -B _build \ +MKLROOT=_build/vcpkg_installed/x64-linux/lib/intel64 cmake -B _build \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_TESTING=OFF \ -DENABLE_PYTHON=OFF \ -DCMAKE_INSTALL_LIBDIR=lib \ -DBLA_VENDOR=Intel10_64lp \ - -DCMAKE_BUILD_TYPE=Release . + -DCMAKE_BUILD_TYPE=Release \ + -DOpenMP_CXX_FLAGS=-fopenmp=libiomp5 \ + -DOpenMP_CXX_LIB_NAMES=libiomp5 \ + -DOpenMP_libiomp5_LIBRARY=$PREFIX/lib/libiomp5.so \ + . -make -C _build -j$(nproc) lintdb +MKLROOT=_build/vcpkg_installed/x64-linux/lib/intel64 make -C _build -j$(nproc) lintdb cmake --install _build --prefix $PREFIX cmake --install _build --prefix _liblintdb_stage/ \ No newline at end of file diff --git a/conda/lintdb/build-pkg.sh b/conda/lintdb/build-pkg.sh index f31e54b..bd25250 100644 --- a/conda/lintdb/build-pkg.sh +++ b/conda/lintdb/build-pkg.sh @@ -4,7 +4,7 @@ set -e # -DPYTHON_INCLUDE_DIR=$(python -c "import sysconfig; print(sysconfig.get_path('include'))") \ # -DPYTHON_LIBRARY=$(python -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))") \ -cmake -B _build_python_${PY_VER} \ +MKLROOT=_build_python_${PY_VER}/vcpkg_installed/x64-linux/lib/intel64 cmake -B _build_python_${PY_VER} \ -DBUILD_SHARED_LIBS=ON \ -DBUILD_TESTING=OFF \ -Dlintdb_ROOT=_liblintdb_stage/ \ @@ -12,7 +12,7 @@ cmake -B _build_python_${PY_VER} \ -DPython_EXECUTABLE=$PYTHON \ . -make -C _build_python_${PY_VER} -j$(nproc) pylintdb +MKLROOT=_build_python_${PY_VER}/vcpkg_installed/x64-linux/lib/intel64 make -C _build_python_${PY_VER} -j$(nproc) pylintdb # Build actual python module. cd _build_python_${PY_VER}/lintdb/python diff --git a/conda/lintdb/meta.yaml b/conda/lintdb/meta.yaml index e9e3fff..a4d47eb 100644 --- a/conda/lintdb/meta.yaml +++ b/conda/lintdb/meta.yaml @@ -37,24 +37,25 @@ outputs: requirements: build: - {{ compiler('cxx') }} - - {{ compiler('fortran')}} + - {{ compiler('fortran') }} - sysroot_linux-64 =2.17 # [linux64] - - llvm-openmp # [osx] - - cmake >=3.23.1 + - llvm-openmp # [osx] + - intel::intel-openmp =2023.0.0 #[linux] + - cmake >=3.25 - make # [not win] - - lapack # [linux] - conda-forge::swig >=4.0.2 - - conda-forge::numpy - - mkl-devel =2023 # [x86_64] - - conda-forge::onnxruntime-cpp + - numpy + - mkl-devel =2023.0.0 # [x86_64] + - conda-forge::onnxruntime-cpp==1.17.3 host: - - mkl =2023 # [x86_64] + - intel::intel-openmp =2023.0.0 #[linux] + - mkl =2023.0.0 # [x86_64] - openblas # [not x86_64] - - conda-forge::onnxruntime-cpp + - conda-forge::onnxruntime-cpp==1.17.3 run: - - mkl =2023 # [x86_64] + - mkl =2023.0.0 # [x86_64] - openblas # [not x86_64] - - conda-forge::onnxruntime-cpp + - conda-forge::onnxruntime-cpp==1.17.3 - __osx >={{ MACOSX_DEPLOYMENT_TARGET|default("10.13") }} # [osx and x86_64] test: requires: @@ -64,7 +65,6 @@ outputs: - conda inspect linkages -p $PREFIX $PKG_NAME # [not win] - conda inspect objects -p $PREFIX $PKG_NAME # [osx] - - name: lintdb script: build-pkg.sh # [x86_64 and linux] script: build-pkg-osx.sh # [x86_64 and osx] @@ -76,28 +76,26 @@ outputs: requirements: build: - {{ compiler('cxx') }} - - {{ compiler('fortran') }} - - llvm-openmp # [osx] - # - lapack # [linux] + - llvm-openmp - sysroot_linux-64 =2.17 # [linux64] - conda-forge::swig >= 4.0.2 - - conda-forge::numpy - - cmake >=3.23.1 + - numpy + - cmake >=3.25 - make # [not win] - python {{ python }} host: - python {{ python }} - - conda-forge::numpy + - numpy - {{ pin_subpackage('liblintdb', exact=True) }} run: - python {{ python }} - - conda-forge::numpy + - numpy - packaging - {{ pin_subpackage('liblintdb', exact=True) }} - __osx >={{ MACOSX_DEPLOYMENT_TARGET|default("10.13") }} # [osx and x86_64] test: requires: - - conda-forge::numpy + - numpy - pytorch commands: - python -X faulthandler -m unittest discover -v -s tests/ -p "test_*" diff --git a/lintdb/python/CMakeLists.txt b/lintdb/python/CMakeLists.txt index 7a61361..ee8fba7 100644 --- a/lintdb/python/CMakeLists.txt +++ b/lintdb/python/CMakeLists.txt @@ -17,7 +17,7 @@ include(${SWIG_USE_FILE}) include(UseSWIG) # below is useful for debugging typemaps. It will log how each object resolves its typemap. -set(CMAKE_SWIG_FLAGS "-debug-tmsearch") +# set(CMAKE_SWIG_FLAGS "-debug-tmsearch") set(UseSWIG_TARGET_NAME_PREFERENCE STANDARD) set(SWIG_SOURCE_FILE_EXTENSIONS swig) @@ -82,4 +82,4 @@ configure_file(setup.py setup.py COPYONLY) configure_file(__init__.py __init__.py COPYONLY) # copy our version file -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.txt ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) \ No newline at end of file +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.txt ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) diff --git a/lintdb/version.h b/lintdb/version.h index 2083fb4..3ebddd9 100644 --- a/lintdb/version.h +++ b/lintdb/version.h @@ -1,6 +1,6 @@ #pragma once -#define LINTDB_VERSION_STRING "0.3.1" +#define LINTDB_VERSION_STRING "0.4.0" namespace lintdb { diff --git a/ports/rocksdb/0001-fix-dependencies.patch b/ports/rocksdb/0001-fix-dependencies.patch new file mode 100644 index 0000000..7c16654 --- /dev/null +++ b/ports/rocksdb/0001-fix-dependencies.patch @@ -0,0 +1,122 @@ + CMakeLists.txt | 33 +++++++++++++++------------------ + cmake/RocksDBConfig.cmake.in | 11 ++++++++--- + 2 files changed, 23 insertions(+), 21 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 23a4014bc..045f5a36d 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -87,7 +87,7 @@ endif() + + include(CMakeDependentOption) + +-if(MSVC) ++if(0) + option(WITH_GFLAGS "build with GFlags" OFF) + option(WITH_XPRESS "build with windows built in compression" OFF) + option(ROCKSDB_SKIP_THIRDPARTY "skip thirdparty.inc" OFF) +@@ -136,10 +136,7 @@ else() + endif() + + if(WITH_SNAPPY) +- find_package(Snappy CONFIG) +- if(NOT Snappy_FOUND) +- find_package(Snappy REQUIRED) +- endif() ++ find_package(Snappy CONFIG REQUIRED) + add_definitions(-DSNAPPY) + list(APPEND THIRDPARTY_LIBS Snappy::snappy) + endif() +@@ -163,16 +160,19 @@ else() + endif() + + if(WITH_LZ4) +- find_package(lz4 REQUIRED) ++ find_package(lz4 CONFIG REQUIRED) + add_definitions(-DLZ4) + list(APPEND THIRDPARTY_LIBS lz4::lz4) + endif() + + if(WITH_ZSTD) +- find_package(zstd REQUIRED) ++ find_package(zstd CONFIG REQUIRED) + add_definitions(-DZSTD) +- include_directories(${ZSTD_INCLUDE_DIR}) +- list(APPEND THIRDPARTY_LIBS zstd::zstd) ++ if(TARGET zstd::libzstd_shared) ++ list(APPEND THIRDPARTY_LIBS zstd::libzstd_shared) ++ elseif(TARGET zstd::libzstd_static) ++ list(APPEND THIRDPARTY_LIBS zstd::libzstd_static) ++ endif() + endif() + endif() + +@@ -312,11 +312,10 @@ int main() { + endif() + + if (WITH_LIBURING) +- find_package(uring) +- if (uring_FOUND) +- add_definitions(-DROCKSDB_IOURING_PRESENT) +- list(APPEND THIRDPARTY_LIBS uring::uring) +- endif() ++ find_package(PkgConfig) ++ pkg_check_modules(liburing REQUIRED IMPORTED_TARGET GLOBAL liburing>=2.0) ++ add_definitions(-DROCKSDB_IOURING_PRESENT) ++ list(APPEND THIRDPARTY_LIBS PkgConfig::liburing) + endif() + + # Reset the required flags +@@ -382,9 +381,9 @@ endif() + + option(WITH_TBB "build with Threading Building Blocks (TBB)" OFF) + if(WITH_TBB) +- find_package(TBB REQUIRED) ++ find_package(TBB CONFIG REQUIRED) + add_definitions(-DTBB) +- list(APPEND THIRDPARTY_LIBS TBB::TBB) ++ list(APPEND THIRDPARTY_LIBS TBB::tbb) + endif() + + # Stall notifications eat some performance from inserts +@@ -1202,8 +1201,6 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS) + endforeach() + endforeach() + +- install(DIRECTORY "${PROJECT_SOURCE_DIR}/cmake/modules" COMPONENT devel DESTINATION ${package_config_destination}) +- + install( + TARGETS ${ROCKSDB_STATIC_LIB} + EXPORT RocksDBTargets +diff --git a/cmake/RocksDBConfig.cmake.in b/cmake/RocksDBConfig.cmake.in +index 0bd14be11..a420d8bfe 100644 +--- a/cmake/RocksDBConfig.cmake.in ++++ b/cmake/RocksDBConfig.cmake.in +@@ -33,11 +33,11 @@ if(@WITH_BZ2@) + endif() + + if(@WITH_LZ4@) +- find_dependency(lz4) ++ find_dependency(lz4 CONFIG) + endif() + + if(@WITH_ZSTD@) +- find_dependency(zstd) ++ find_dependency(zstd CONFIG) + endif() + + if(@WITH_NUMA@) +@@ -45,7 +45,12 @@ if(@WITH_NUMA@) + endif() + + if(@WITH_TBB@) +- find_dependency(TBB) ++ find_dependency(TBB CONFIG) ++endif() ++ ++if(@WITH_LIBURING@) ++ find_dependency(PkgConfig) ++ pkg_check_modules(liburing REQUIRED IMPORTED_TARGET GLOBAL liburing>=2.0) + endif() + + find_dependency(Threads) diff --git a/ports/rocksdb/portfile.cmake b/ports/rocksdb/portfile.cmake new file mode 100644 index 0000000..deb1cf0 --- /dev/null +++ b/ports/rocksdb/portfile.cmake @@ -0,0 +1,68 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO facebook/rocksdb + REF "v${VERSION}" + SHA512 524e3e70ed2b1d2e6c61a7b401946e50473cc95684ce4efc6250062f5bc945e443e96f7907fcc3ee1ab98c71179a8b56a654383cf2c0bbe1bb20907ab1ac7523 + HEAD_REF main + PATCHES + 0001-fix-dependencies.patch +) + +string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" WITH_MD_LIBRARY) +string(COMPARE EQUAL "${VCPKG_LIBRARY_LINKAGE}" "dynamic" ROCKSDB_BUILD_SHARED) + +vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + "liburing" WITH_LIBURING + "snappy" WITH_SNAPPY + "lz4" WITH_LZ4 + "zlib" WITH_ZLIB + "zstd" WITH_ZSTD + "bzip2" WITH_BZ2 + "numa" WITH_NUMA + "tbb" WITH_TBB +) + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + -DWITH_GFLAGS=OFF + -DWITH_TESTS=OFF + -DWITH_BENCHMARK_TOOLS=OFF + -DWITH_TOOLS=OFF + -DUSE_RTTI=ON + -DROCKSDB_INSTALL_ON_WINDOWS=ON + -DFAIL_ON_WARNINGS=OFF + -DWITH_MD_LIBRARY=${WITH_MD_LIBRARY} + -DPORTABLE=1 # Minimum CPU arch to support, or 0 = current CPU, 1 = baseline CPU + -DROCKSDB_BUILD_SHARED=${ROCKSDB_BUILD_SHARED} + -DCMAKE_DISABLE_FIND_PACKAGE_Git=TRUE + ${FEATURE_OPTIONS} + OPTIONS_DEBUG + -DCMAKE_DEBUG_POSTFIX=d + -DWITH_RUNTIME_DEBUG=ON + OPTIONS_RELEASE + -DWITH_RUNTIME_DEBUG=OFF +) + +vcpkg_cmake_install() + +vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/rocksdb) + +vcpkg_copy_pdbs() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") + +vcpkg_fixup_pkgconfig() + +vcpkg_install_copyright(COMMENT [[ +RocksDB is dual-licensed under both the GPLv2 (found in COPYING) +and Apache 2.0 License (found in LICENSE.Apache). You may select, +at your option, one of the above-listed licenses. +]] + FILE_LIST + "${SOURCE_PATH}/LICENSE.leveldb" + "${SOURCE_PATH}/LICENSE.Apache" + "${SOURCE_PATH}/COPYING" +) diff --git a/ports/rocksdb/vcpkg.json b/ports/rocksdb/vcpkg.json new file mode 100644 index 0000000..5d1bf37 --- /dev/null +++ b/ports/rocksdb/vcpkg.json @@ -0,0 +1,73 @@ +{ + "name": "rocksdb", + "version": "8.10.0", + "description": "A library that provides an embeddable, persistent key-value store for fast storage", + "homepage": "https://github.com/facebook/rocksdb", + "license": "GPL-2.0-only OR Apache-2.0", + "supports": "!uwp & !(arm & !arm64 & android)", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "zlib" + ], + "features": { + "bzip2": { + "description": "build with bzip2", + "dependencies": [ + "bzip2" + ] + }, + "liburing": { + "description": "build with liburing", + "supports": "linux", + "dependencies": [ + { + "name": "liburing", + "platform": "linux" + } + ] + }, + "lz4": { + "description": "build with lz4", + "dependencies": [ + "lz4" + ] + }, + "numa": { + "description": "build with NUMA policy support", + "supports": "linux" + }, + "snappy": { + "description": "build with SNAPPY", + "dependencies": [ + "snappy" + ] + }, + "tbb": { + "description": "build with Threading Building Blocks (TBB)", + "dependencies": [ + "tbb" + ] + }, + "zlib": { + "description": "build with zlib", + "dependencies": [ + "zlib" + ] + }, + "zstd": { + "description": "build with zstd", + "dependencies": [ + "zstd" + ] + } + } +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5125c51..2553ff1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,7 +1,7 @@ set( LINT_DB_TESTS - util.h - index_test.cpp + util.h + index_test.cpp plaid_test.cpp rocksdb_test.cpp binarizer_test.cpp @@ -9,8 +9,8 @@ set( index_builder_test.cpp mocks.h xtr_test.cpp - inverted_list_test.cpp - encoded_doc_test.cpp + inverted_list_test.cpp + encoded_doc_test.cpp ) add_executable( diff --git a/vcpkg.json b/vcpkg.json index 9ae68f8..2c33af7 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,6 +1,6 @@ { "name": "lintdb", - "version-string": "0.3.1", + "version-string": "0.4.0", "license": "MIT", "dependencies": [ "faiss", diff --git a/version.txt b/version.txt index a2268e2..60a2d3e 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.3.1 \ No newline at end of file +0.4.0 \ No newline at end of file