From 7c698ff00a09b0081c3d3a8c93cb850dbaf9d9c7 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sat, 20 Feb 2021 10:45:30 +0100 Subject: [PATCH 01/12] Revert "disable win + cuda until debugged fully" This reverts commit be141cf36746263e29bdf7ea36b19923627cec6a. --- recipe/meta.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 730a9cb4..5811f1f4 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -103,8 +103,6 @@ build: number: {{ number }} # GPU version for linux64 & win skip: true # [osx and cuda_compiler_version != "None"] - # DEBUG: temporarily disable win + cuda - skip: true # [win and cuda_compiler_version != "None"] requirements: build: From d6aa7bc4633b168b720073aecdf7ea6a6bbb89a0 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 24 Feb 2021 23:41:47 +0100 Subject: [PATCH 02/12] stay on C++11 for win + cuda 10.2 --- recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 5811f1f4..8207affe 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -71,7 +71,7 @@ source: url: https://github.com/facebookresearch/faiss/archive/v{{ version }}.tar.gz sha256: f86d346ac9f409ee30abe37e52f6cce366b7f60d3924d65719f40aa07ceb4bec patches: - - patches/0001-use-c-14.patch + - patches/0001-use-c-14.patch # [win and cuda_compiler_version != "10.2"] # backport of facebookresearch/faiss#1666, can be dropped for ver>1.7.0 - patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch # update version-guard to build for compute_86, can be dropped for ver>1.7.0 From 2e4e4f8f11ba3cf766e8b967eb6b3bb50bbf8a04 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Mar 2021 19:55:29 +0100 Subject: [PATCH 03/12] add rebased win_cuda patches --- recipe/meta.yaml | 10 ++- recipe/patches/0001-use-c-14.patch | 2 +- ...ers-in-faiss-gpu-CMakeLists.txt-1666.patch | 2 +- ...003-update-util-guard-for-compute_86.patch | 2 +- ...missing-includes-for-std-min-std-max.patch | 2 +- ...-test-that-fails-without-GPU-drivers.patch | 2 +- ...-detection-platform-independent-1600.patch | 2 +- ...ke-setup.py-win-avx2-compatible-1682.patch | 2 +- ...essages-and-errors-in-loader.py-1683.patch | 2 +- ...ble-AVX2-switch-in-CMakeLists.txt-16.patch | 2 +- .../patches/0010-Win-AVX2-compat-1681.patch | 2 +- ...011-Windows-portable-intrinsics-1684.patch | 2 +- ...eader-target-directory-to-outputname.patch | 2 +- .../patches/0013-Fix-swig-vs.-AVX2-1717.patch | 81 +++++++++++++++++++ ...orrectly-for-AVX2-in-swigfaiss-build.patch | 27 ------- ...ll-fixes-for-compilation-on-ARM-1655.patch | 4 +- ...erload-__builtin_ctz-for-win-cuda-11.patch | 31 +++++++ ...fix-int32-vs-int64-issue-in-GPU-test.patch | 30 +++++++ ...-skip-test_stress-for-GPU-on-windows.patch | 32 ++++++++ 19 files changed, 196 insertions(+), 43 deletions(-) create mode 100644 recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch delete mode 100644 recipe/patches/0013-switch-correctly-for-AVX2-in-swigfaiss-build.patch create mode 100644 recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch create mode 100644 recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch create mode 100644 recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 8207affe..9fc714d7 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -94,10 +94,16 @@ source: - patches/0011-Windows-portable-intrinsics-1684.patch # adapt header target directory for faiss_avx2 - patches/0012-adapt-header-target-directory-to-outputname.patch - # correctly switch for avx2 in swig-build, see faiss#1717 - - patches/0013-switch-correctly-for-AVX2-in-swigfaiss-build.patch + # backport of facebookresearch/faiss#1717, can be dropped for ver>1.7.0 + - patches/0013-Fix-swig-vs.-AVX2-1717.patch # backport fix for flaky test under AVX2; from faiss#1655, can be dropped for ver>1.7.0 - patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch + # avoid warning on win + cuda >= 11 + - patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch + # fix GPU test on windows + - patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch + # patch for avoiding crash in GPU test suite on windows + - patches/0017-skip-test_stress-for-GPU-on-windows.patch build: number: {{ number }} diff --git a/recipe/patches/0001-use-c-14.patch b/recipe/patches/0001-use-c-14.patch index 1de7c8a4..6fea16da 100644 --- a/recipe/patches/0001-use-c-14.patch +++ b/recipe/patches/0001-use-c-14.patch @@ -1,7 +1,7 @@ From 1b4e6f16de1bc6e6e7a104647625f45956356df5 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 26 Oct 2020 22:44:44 +0100 -Subject: [PATCH 01/14] use c++14 +Subject: [PATCH 01/17] use c++14 --- CMakeLists.txt | 2 +- diff --git a/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch b/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch index 4b6fa2e1..e317c7c8 100644 --- a/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch +++ b/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch @@ -1,7 +1,7 @@ From e4b94c8a5ad0fc7632cf05006865dbd8feac2ed4 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Feb 2021 09:21:21 -0800 -Subject: [PATCH 02/14] Add missing headers in faiss/[gpu/]CMakeLists.txt +Subject: [PATCH 02/17] Add missing headers in faiss/[gpu/]CMakeLists.txt (#1666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 diff --git a/recipe/patches/0003-update-util-guard-for-compute_86.patch b/recipe/patches/0003-update-util-guard-for-compute_86.patch index 280d0cd2..d280b7dc 100644 --- a/recipe/patches/0003-update-util-guard-for-compute_86.patch +++ b/recipe/patches/0003-update-util-guard-for-compute_86.patch @@ -1,7 +1,7 @@ From 84b98debfb92145a55e932aeb74eb0358ed15512 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 18:16:11 +0100 -Subject: [PATCH 03/14] update util-guard for compute_86 +Subject: [PATCH 03/17] update util-guard for compute_86 --- faiss/gpu/utils/DeviceDefs.cuh | 2 +- diff --git a/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch b/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch index f5d25839..651c09ce 100644 --- a/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch +++ b/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch @@ -1,7 +1,7 @@ From c76bdcca995efee62144f0efbfdff1d7c7c3957c Mon Sep 17 00:00:00 2001 From: Lucas Hosseini Date: Thu, 7 Jan 2021 12:21:42 +0100 -Subject: [PATCH 04/14] Add missing includes for std::min/std::max. +Subject: [PATCH 04/17] Add missing includes for std::min/std::max. --- faiss/gpu/GpuIndex.cu | 1 + diff --git a/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch b/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch index 26fc9899..ed9d823b 100644 --- a/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch +++ b/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch @@ -1,7 +1,7 @@ From 1b4f509cdc91d38fecca58709093748324821931 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 18:36:12 +0100 -Subject: [PATCH 05/14] skip test that fails without GPU drivers +Subject: [PATCH 05/17] skip test that fails without GPU drivers --- tests/test_contrib.py | 3 +-- diff --git a/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch b/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch index f4d2697b..84819c72 100644 --- a/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch +++ b/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch @@ -1,7 +1,7 @@ From c329c59103f90a9b5fcec98f7a71f1bba4a3edbc Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 3 Feb 2021 08:00:39 -0800 -Subject: [PATCH 06/14] make AVX2-detection platform-independent (#1600) +Subject: [PATCH 06/17] make AVX2-detection platform-independent (#1600) Summary: In the context of https://github.com/conda-forge/faiss-split-feedstock/issues/23, I discussed with some of the conda-folks how we should support AVX2 (and potentially other builds) for faiss. In the meantime, we'd like to follow the model that faiss itself is using (i.e. build with AVX2 and without and then load the corresponding library at runtime depending on CPU capabilities). diff --git a/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch b/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch index 38fec3ce..7c4c6318 100644 --- a/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch +++ b/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch @@ -1,7 +1,7 @@ From 4888d088f4bd62e82ddddd54fa98565ae201c566 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 15:07:28 -0800 -Subject: [PATCH 07/14] make setup.py win+avx2 compatible (#1682) +Subject: [PATCH 07/17] make setup.py win+avx2 compatible (#1682) Summary: While working on https://github.com/conda-forge/faiss-split-feedstock/pull/27, it turned out I needed diff --git a/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch b/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch index 4f9ba638..fa83da38 100644 --- a/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch +++ b/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch @@ -1,7 +1,7 @@ From af260092ea6fb23c61cea1d4b239da0334ad341c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 12 Feb 2021 10:57:44 -0800 -Subject: [PATCH 08/14] log success messages and errors in loader.py (#1683) +Subject: [PATCH 08/17] log success messages and errors in loader.py (#1683) Summary: This patch is to provide log information in case of failing to load the AVX2-enabled lib, diff --git a/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch b/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch index 6d3d6fc8..5468d1b5 100644 --- a/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch +++ b/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch @@ -1,7 +1,7 @@ From 6b361d39aa12da3cc18b4c75317707bfa67dc1a1 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 14:05:11 -0800 -Subject: [PATCH 09/14] add msvc-compatible AVX2 switch in CMakeLists.txt +Subject: [PATCH 09/17] add msvc-compatible AVX2 switch in CMakeLists.txt (#1680) Summary: diff --git a/recipe/patches/0010-Win-AVX2-compat-1681.patch b/recipe/patches/0010-Win-AVX2-compat-1681.patch index 793edf81..8770433c 100644 --- a/recipe/patches/0010-Win-AVX2-compat-1681.patch +++ b/recipe/patches/0010-Win-AVX2-compat-1681.patch @@ -1,7 +1,7 @@ From 0e4bea421e47414853ff2345bc2438d22769424b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 08:04:30 -0800 -Subject: [PATCH 10/14] Win+AVX2 compat (#1681) +Subject: [PATCH 10/17] Win+AVX2 compat (#1681) Summary: Upstreaming patches from conda-forge/faiss-split-feedstock#27, follow-up (sorta) to https://github.com/facebookresearch/faiss/issues/1600. diff --git a/recipe/patches/0011-Windows-portable-intrinsics-1684.patch b/recipe/patches/0011-Windows-portable-intrinsics-1684.patch index 39a22981..b29ad3dd 100644 --- a/recipe/patches/0011-Windows-portable-intrinsics-1684.patch +++ b/recipe/patches/0011-Windows-portable-intrinsics-1684.patch @@ -1,7 +1,7 @@ From a987f2cf36de655be6273e7c91867c0dfedf09a9 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 Feb 2021 11:27:33 -0800 -Subject: [PATCH 11/14] Windows: portable intrinsics (#1684) +Subject: [PATCH 11/17] Windows: portable intrinsics (#1684) Summary: Trying to compile windows for AVX2 in https://github.com/conda-forge/faiss-split-feedstock/pull/27 diff --git a/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch b/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch index 40a84efc..6274bc22 100644 --- a/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch +++ b/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch @@ -1,7 +1,7 @@ From 84072b99d78be9d94bf902e66fff396c1b94b21b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 14 Feb 2021 15:52:27 +0100 -Subject: [PATCH 12/14] adapt header target directory to outputname +Subject: [PATCH 12/17] adapt header target directory to outputname --- faiss/CMakeLists.txt | 11 ++++++++--- diff --git a/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch b/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch new file mode 100644 index 00000000..eedf320c --- /dev/null +++ b/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch @@ -0,0 +1,81 @@ +From 95e146d75efbbde1b55d5a9ae8f3957c6574ae5c Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Thu, 4 Mar 2021 03:43:07 -0800 +Subject: [PATCH 13/17] Fix swig vs. AVX2 (#1717) + +Summary: +Towards https://github.com/facebookresearch/faiss/issues/1711. There's still a test [error](https://github.com/facebookresearch/faiss/issues/1711#issuecomment-787509873) that won't be picked up unless an AVX2-specific CI run is added. + +Pull Request resolved: https://github.com/facebookresearch/faiss/pull/1717 + +Reviewed By: mdouze + +Differential Revision: D26814113 + +Pulled By: beauby + +fbshipit-source-id: c3e64a04c3afd1d354439a2c4a2edd611d2c6af8 +--- + conda/faiss-gpu/build-pkg.sh | 3 ++- + conda/faiss/build-pkg.sh | 3 ++- + faiss/python/CMakeLists.txt | 4 ++++ + 3 files changed, 8 insertions(+), 2 deletions(-) + +diff --git a/conda/faiss-gpu/build-pkg.sh b/conda/faiss-gpu/build-pkg.sh +index c3aaca65..dfb59425 100755 +--- a/conda/faiss-gpu/build-pkg.sh ++++ b/conda/faiss-gpu/build-pkg.sh +@@ -10,6 +10,7 @@ set -e + # Build avx2 version. + cmake -B _build_python_${PY_VER}_avx2 \ + -Dfaiss_ROOT=_libfaiss_avx2_stage/ \ ++ -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_ENABLE_GPU=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython_EXECUTABLE=$PYTHON \ +@@ -31,6 +32,6 @@ cmake --build _build_python_${PY_VER} -j $CPU_COUNT + + # Build actual python module. + cp _build_python_${PY_VER}_avx2/swigfaiss.py _build_python_${PY_VER}/swigfaiss_avx2.py +-cp _build_python_${PY_VER}_avx2/_swigfaiss.so _build_python_${PY_VER}/_swigfaiss_avx2.so ++cp _build_python_${PY_VER}_avx2/_swigfaiss_avx2.so _build_python_${PY_VER}/_swigfaiss_avx2.so + cd _build_python_${PY_VER}/ + $PYTHON setup.py install --single-version-externally-managed --record=record.txt --prefix=$PREFIX +diff --git a/conda/faiss/build-pkg.sh b/conda/faiss/build-pkg.sh +index 541537bd..db9b0949 100755 +--- a/conda/faiss/build-pkg.sh ++++ b/conda/faiss/build-pkg.sh +@@ -10,6 +10,7 @@ set -e + # Build avx2 version. + cmake -B _build_python_${PY_VER}_avx2 \ + -Dfaiss_ROOT=_libfaiss_avx2_stage/ \ ++ -DFAISS_OPT_LEVEL=avx2 \ + -DFAISS_ENABLE_GPU=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython_EXECUTABLE=$PYTHON \ +@@ -31,6 +32,6 @@ cmake --build _build_python_${PY_VER} -j $CPU_COUNT + + # Build actual python module. + cp _build_python_${PY_VER}_avx2/swigfaiss.py _build_python_${PY_VER}/swigfaiss_avx2.py +-cp _build_python_${PY_VER}_avx2/_swigfaiss.so _build_python_${PY_VER}/_swigfaiss_avx2.so ++cp _build_python_${PY_VER}_avx2/_swigfaiss_avx2.so _build_python_${PY_VER}/_swigfaiss_avx2.so + cd _build_python_${PY_VER}/ + $PYTHON setup.py install --single-version-externally-managed --record=record.txt --prefix=$PREFIX +diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt +index 40ef40c1..2ed9c200 100644 +--- a/faiss/python/CMakeLists.txt ++++ b/faiss/python/CMakeLists.txt +@@ -61,6 +61,10 @@ swig_add_library(swigfaiss + SOURCES swigfaiss.swig + ) + ++if(FAISS_OPT_LEVEL STREQUAL "avx2") ++ set_target_properties(swigfaiss PROPERTIES OUTPUT_NAME "swigfaiss_avx2") ++endif() ++ + if(NOT WIN32) + # NOTE: Python does not recognize the dylib extension. + set_target_properties(swigfaiss PROPERTIES SUFFIX .so) +-- +2.29.2.windows.3 + diff --git a/recipe/patches/0013-switch-correctly-for-AVX2-in-swigfaiss-build.patch b/recipe/patches/0013-switch-correctly-for-AVX2-in-swigfaiss-build.patch deleted file mode 100644 index 82b4a677..00000000 --- a/recipe/patches/0013-switch-correctly-for-AVX2-in-swigfaiss-build.patch +++ /dev/null @@ -1,27 +0,0 @@ -From d49343b508f98e48dd34ece4d340f878b4a89d87 Mon Sep 17 00:00:00 2001 -From: "H. Vetinari" -Date: Sun, 28 Feb 2021 00:00:57 +0100 -Subject: [PATCH 13/14] switch correctly for AVX2 in swigfaiss-build - ---- - faiss/python/CMakeLists.txt | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt -index 40ef40c1..2ed9c200 100644 ---- a/faiss/python/CMakeLists.txt -+++ b/faiss/python/CMakeLists.txt -@@ -61,6 +61,10 @@ swig_add_library(swigfaiss - SOURCES swigfaiss.swig - ) - -+if(FAISS_OPT_LEVEL STREQUAL "avx2") -+ set_target_properties(swigfaiss PROPERTIES OUTPUT_NAME "swigfaiss_avx2") -+endif() -+ - if(NOT WIN32) - # NOTE: Python does not recognize the dylib extension. - set_target_properties(swigfaiss PROPERTIES SUFFIX .so) --- -2.29.2.windows.3 - diff --git a/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch b/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch index 0e7cc88e..cb86d4cd 100644 --- a/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch +++ b/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch @@ -1,7 +1,7 @@ -From 7ef0cedd2564fe608946489249c9901fef746bea Mon Sep 17 00:00:00 2001 +From cab8a50faddf8413322c1646a8326e4c68f52405 Mon Sep 17 00:00:00 2001 From: Matthijs Douze Date: Fri, 29 Jan 2021 10:05:07 -0800 -Subject: [PATCH 14/14] Small fixes for compilation on ARM (#1655) +Subject: [PATCH 14/17] Small fixes for compilation on ARM (#1655) Summary: This PR fixes a few small issues with compilation on ARM. diff --git a/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch b/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch new file mode 100644 index 00000000..a82508fe --- /dev/null +++ b/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch @@ -0,0 +1,31 @@ +From 1c0e08310b0ab7afb558f987adfdf3bbcca3085c Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Thu, 18 Feb 2021 19:30:58 +0100 +Subject: [PATCH 15/17] do not overload __builtin_ctz for win + cuda>=11 + +--- + faiss/impl/platform_macros.h | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/faiss/impl/platform_macros.h b/faiss/impl/platform_macros.h +index 1f4795c6..7efc9483 100644 +--- a/faiss/impl/platform_macros.h ++++ b/faiss/impl/platform_macros.h +@@ -39,11 +39,14 @@ inline int __builtin_ctzll(uint64_t x) { + return (int)ret; + } + ++// cudatoolkit provides __builtin_ctz for NVCC >= 11.0 ++#if ! defined(__CUDACC__) || __CUDACC_VER_MAJOR__ < 11 + inline int __builtin_ctz(unsigned long x) { + unsigned long ret; + _BitScanForward(&ret, x); + return (int)ret; + } ++#endif + + inline int __builtin_clzll(uint64_t x) { + return (int)__lzcnt64(x); +-- +2.29.2.windows.3 + diff --git a/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch b/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch new file mode 100644 index 00000000..fe26e594 --- /dev/null +++ b/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch @@ -0,0 +1,30 @@ +From 014f5b4ef37274cdbf295e336bea78c62d87fb7c Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Fri, 19 Feb 2021 19:07:40 +0100 +Subject: [PATCH 16/17] fix int32 vs int64 issue in GPU test + +--- + faiss/gpu/test/test_gpu_index.py | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/faiss/gpu/test/test_gpu_index.py b/faiss/gpu/test/test_gpu_index.py +index d96bd13a..d5af7a46 100755 +--- a/faiss/gpu/test/test_gpu_index.py ++++ b/faiss/gpu/test/test_gpu_index.py +@@ -61,8 +61,11 @@ class EvalIVFPQAccuracy(unittest.TestCase): + index.train(xt) + ts.append(time.time()) + +- # adding some ids because there was a bug in this case +- index.add_with_ids(xb, np.arange(nb) * 3 + 12345) ++ # adding some ids because there was a bug in this case; ++ # those need to be cast to idx_t(= int64_t), because ++ # on windows the numpy int default is int32 ++ ids = (np.arange(nb) * 3 + 12345).astype('int64') ++ index.add_with_ids(xb, ids) + ts.append(time.time()) + + index.nprobe = 4 +-- +2.29.2.windows.3 + diff --git a/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch b/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch new file mode 100644 index 00000000..df907cc8 --- /dev/null +++ b/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch @@ -0,0 +1,32 @@ +From eda6e7f7a743e7250162f03e5b3d825e65e3c167 Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Fri, 19 Feb 2021 14:14:31 +0100 +Subject: [PATCH 17/17] skip test_stress for GPU on windows + +--- + faiss/gpu/test/test_gpu_index.py | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/faiss/gpu/test/test_gpu_index.py b/faiss/gpu/test/test_gpu_index.py +index d5af7a46..43084bff 100755 +--- a/faiss/gpu/test/test_gpu_index.py ++++ b/faiss/gpu/test/test_gpu_index.py +@@ -10,6 +10,7 @@ import unittest + import numpy as np + import faiss + import math ++import sys + + class EvalIVFPQAccuracy(unittest.TestCase): + +@@ -202,6 +203,7 @@ class ReferencedObject(unittest.TestCase): + + assert ref == new + ++ @unittest.skipIf(sys.platform == 'win32', 'MemoryError!') + def test_stress(self): + # a mixture of the above, from issue #631 + target = np.random.rand(50, 16).astype('float32') +-- +2.29.2.windows.3 + From c11a4756086123629a98d6b1638436a7f6ab547d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Mar 2021 19:57:00 +0100 Subject: [PATCH 04/12] add patch for long vs. windows in swigfaiss.swig --- recipe/meta.yaml | 2 + recipe/patches/0001-use-c-14.patch | 2 +- ...ers-in-faiss-gpu-CMakeLists.txt-1666.patch | 2 +- ...003-update-util-guard-for-compute_86.patch | 2 +- ...missing-includes-for-std-min-std-max.patch | 2 +- ...-test-that-fails-without-GPU-drivers.patch | 2 +- ...-detection-platform-independent-1600.patch | 2 +- ...ke-setup.py-win-avx2-compatible-1682.patch | 2 +- ...essages-and-errors-in-loader.py-1683.patch | 2 +- ...ble-AVX2-switch-in-CMakeLists.txt-16.patch | 2 +- .../patches/0010-Win-AVX2-compat-1681.patch | 2 +- ...011-Windows-portable-intrinsics-1684.patch | 2 +- ...eader-target-directory-to-outputname.patch | 2 +- .../patches/0013-Fix-swig-vs.-AVX2-1717.patch | 2 +- ...ll-fixes-for-compilation-on-ARM-1655.patch | 2 +- ...erload-__builtin_ctz-for-win-cuda-11.patch | 2 +- ...fix-int32-vs-int64-issue-in-GPU-test.patch | 2 +- ...-skip-test_stress-for-GPU-on-windows.patch | 2 +- ...e-long-vs.-windows-in-swigfaiss.swig.patch | 127 ++++++++++++++++++ 19 files changed, 146 insertions(+), 17 deletions(-) create mode 100644 recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 9fc714d7..3a317f28 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -104,6 +104,8 @@ source: - patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch # patch for avoiding crash in GPU test suite on windows - patches/0017-skip-test_stress-for-GPU-on-windows.patch + # exploratory patch for long vs. windows in swigfaiss.swig + - patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch build: number: {{ number }} diff --git a/recipe/patches/0001-use-c-14.patch b/recipe/patches/0001-use-c-14.patch index 6fea16da..4ef1946b 100644 --- a/recipe/patches/0001-use-c-14.patch +++ b/recipe/patches/0001-use-c-14.patch @@ -1,7 +1,7 @@ From 1b4e6f16de1bc6e6e7a104647625f45956356df5 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 26 Oct 2020 22:44:44 +0100 -Subject: [PATCH 01/17] use c++14 +Subject: [PATCH 01/18] use c++14 --- CMakeLists.txt | 2 +- diff --git a/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch b/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch index e317c7c8..bd0f8f21 100644 --- a/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch +++ b/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch @@ -1,7 +1,7 @@ From e4b94c8a5ad0fc7632cf05006865dbd8feac2ed4 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Feb 2021 09:21:21 -0800 -Subject: [PATCH 02/17] Add missing headers in faiss/[gpu/]CMakeLists.txt +Subject: [PATCH 02/18] Add missing headers in faiss/[gpu/]CMakeLists.txt (#1666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 diff --git a/recipe/patches/0003-update-util-guard-for-compute_86.patch b/recipe/patches/0003-update-util-guard-for-compute_86.patch index d280b7dc..cc89ac9f 100644 --- a/recipe/patches/0003-update-util-guard-for-compute_86.patch +++ b/recipe/patches/0003-update-util-guard-for-compute_86.patch @@ -1,7 +1,7 @@ From 84b98debfb92145a55e932aeb74eb0358ed15512 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 18:16:11 +0100 -Subject: [PATCH 03/17] update util-guard for compute_86 +Subject: [PATCH 03/18] update util-guard for compute_86 --- faiss/gpu/utils/DeviceDefs.cuh | 2 +- diff --git a/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch b/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch index 651c09ce..4db0e635 100644 --- a/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch +++ b/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch @@ -1,7 +1,7 @@ From c76bdcca995efee62144f0efbfdff1d7c7c3957c Mon Sep 17 00:00:00 2001 From: Lucas Hosseini Date: Thu, 7 Jan 2021 12:21:42 +0100 -Subject: [PATCH 04/17] Add missing includes for std::min/std::max. +Subject: [PATCH 04/18] Add missing includes for std::min/std::max. --- faiss/gpu/GpuIndex.cu | 1 + diff --git a/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch b/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch index ed9d823b..62fa33bf 100644 --- a/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch +++ b/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch @@ -1,7 +1,7 @@ From 1b4f509cdc91d38fecca58709093748324821931 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 18:36:12 +0100 -Subject: [PATCH 05/17] skip test that fails without GPU drivers +Subject: [PATCH 05/18] skip test that fails without GPU drivers --- tests/test_contrib.py | 3 +-- diff --git a/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch b/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch index 84819c72..f1e05dc3 100644 --- a/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch +++ b/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch @@ -1,7 +1,7 @@ From c329c59103f90a9b5fcec98f7a71f1bba4a3edbc Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 3 Feb 2021 08:00:39 -0800 -Subject: [PATCH 06/17] make AVX2-detection platform-independent (#1600) +Subject: [PATCH 06/18] make AVX2-detection platform-independent (#1600) Summary: In the context of https://github.com/conda-forge/faiss-split-feedstock/issues/23, I discussed with some of the conda-folks how we should support AVX2 (and potentially other builds) for faiss. In the meantime, we'd like to follow the model that faiss itself is using (i.e. build with AVX2 and without and then load the corresponding library at runtime depending on CPU capabilities). diff --git a/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch b/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch index 7c4c6318..6b598d69 100644 --- a/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch +++ b/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch @@ -1,7 +1,7 @@ From 4888d088f4bd62e82ddddd54fa98565ae201c566 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 15:07:28 -0800 -Subject: [PATCH 07/17] make setup.py win+avx2 compatible (#1682) +Subject: [PATCH 07/18] make setup.py win+avx2 compatible (#1682) Summary: While working on https://github.com/conda-forge/faiss-split-feedstock/pull/27, it turned out I needed diff --git a/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch b/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch index fa83da38..a488f3f6 100644 --- a/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch +++ b/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch @@ -1,7 +1,7 @@ From af260092ea6fb23c61cea1d4b239da0334ad341c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 12 Feb 2021 10:57:44 -0800 -Subject: [PATCH 08/17] log success messages and errors in loader.py (#1683) +Subject: [PATCH 08/18] log success messages and errors in loader.py (#1683) Summary: This patch is to provide log information in case of failing to load the AVX2-enabled lib, diff --git a/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch b/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch index 5468d1b5..b562d0f1 100644 --- a/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch +++ b/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch @@ -1,7 +1,7 @@ From 6b361d39aa12da3cc18b4c75317707bfa67dc1a1 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 14:05:11 -0800 -Subject: [PATCH 09/17] add msvc-compatible AVX2 switch in CMakeLists.txt +Subject: [PATCH 09/18] add msvc-compatible AVX2 switch in CMakeLists.txt (#1680) Summary: diff --git a/recipe/patches/0010-Win-AVX2-compat-1681.patch b/recipe/patches/0010-Win-AVX2-compat-1681.patch index 8770433c..73b76c13 100644 --- a/recipe/patches/0010-Win-AVX2-compat-1681.patch +++ b/recipe/patches/0010-Win-AVX2-compat-1681.patch @@ -1,7 +1,7 @@ From 0e4bea421e47414853ff2345bc2438d22769424b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 08:04:30 -0800 -Subject: [PATCH 10/17] Win+AVX2 compat (#1681) +Subject: [PATCH 10/18] Win+AVX2 compat (#1681) Summary: Upstreaming patches from conda-forge/faiss-split-feedstock#27, follow-up (sorta) to https://github.com/facebookresearch/faiss/issues/1600. diff --git a/recipe/patches/0011-Windows-portable-intrinsics-1684.patch b/recipe/patches/0011-Windows-portable-intrinsics-1684.patch index b29ad3dd..48c32386 100644 --- a/recipe/patches/0011-Windows-portable-intrinsics-1684.patch +++ b/recipe/patches/0011-Windows-portable-intrinsics-1684.patch @@ -1,7 +1,7 @@ From a987f2cf36de655be6273e7c91867c0dfedf09a9 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 Feb 2021 11:27:33 -0800 -Subject: [PATCH 11/17] Windows: portable intrinsics (#1684) +Subject: [PATCH 11/18] Windows: portable intrinsics (#1684) Summary: Trying to compile windows for AVX2 in https://github.com/conda-forge/faiss-split-feedstock/pull/27 diff --git a/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch b/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch index 6274bc22..295e6d5e 100644 --- a/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch +++ b/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch @@ -1,7 +1,7 @@ From 84072b99d78be9d94bf902e66fff396c1b94b21b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 14 Feb 2021 15:52:27 +0100 -Subject: [PATCH 12/17] adapt header target directory to outputname +Subject: [PATCH 12/18] adapt header target directory to outputname --- faiss/CMakeLists.txt | 11 ++++++++--- diff --git a/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch b/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch index eedf320c..c0de8967 100644 --- a/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch +++ b/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch @@ -1,7 +1,7 @@ From 95e146d75efbbde1b55d5a9ae8f3957c6574ae5c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Mar 2021 03:43:07 -0800 -Subject: [PATCH 13/17] Fix swig vs. AVX2 (#1717) +Subject: [PATCH 13/18] Fix swig vs. AVX2 (#1717) Summary: Towards https://github.com/facebookresearch/faiss/issues/1711. There's still a test [error](https://github.com/facebookresearch/faiss/issues/1711#issuecomment-787509873) that won't be picked up unless an AVX2-specific CI run is added. diff --git a/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch b/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch index cb86d4cd..86a80d3d 100644 --- a/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch +++ b/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch @@ -1,7 +1,7 @@ From cab8a50faddf8413322c1646a8326e4c68f52405 Mon Sep 17 00:00:00 2001 From: Matthijs Douze Date: Fri, 29 Jan 2021 10:05:07 -0800 -Subject: [PATCH 14/17] Small fixes for compilation on ARM (#1655) +Subject: [PATCH 14/18] Small fixes for compilation on ARM (#1655) Summary: This PR fixes a few small issues with compilation on ARM. diff --git a/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch b/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch index a82508fe..c1df84c6 100644 --- a/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch +++ b/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch @@ -1,7 +1,7 @@ From 1c0e08310b0ab7afb558f987adfdf3bbcca3085c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 Feb 2021 19:30:58 +0100 -Subject: [PATCH 15/17] do not overload __builtin_ctz for win + cuda>=11 +Subject: [PATCH 15/18] do not overload __builtin_ctz for win + cuda>=11 --- faiss/impl/platform_macros.h | 3 +++ diff --git a/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch b/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch index fe26e594..3b32816c 100644 --- a/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch +++ b/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch @@ -1,7 +1,7 @@ From 014f5b4ef37274cdbf295e336bea78c62d87fb7c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 19 Feb 2021 19:07:40 +0100 -Subject: [PATCH 16/17] fix int32 vs int64 issue in GPU test +Subject: [PATCH 16/18] fix int32 vs int64 issue in GPU test --- faiss/gpu/test/test_gpu_index.py | 7 +++++-- diff --git a/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch b/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch index df907cc8..0264360c 100644 --- a/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch +++ b/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch @@ -1,7 +1,7 @@ From eda6e7f7a743e7250162f03e5b3d825e65e3c167 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 19 Feb 2021 14:14:31 +0100 -Subject: [PATCH 17/17] skip test_stress for GPU on windows +Subject: [PATCH 17/18] skip test_stress for GPU on windows --- faiss/gpu/test/test_gpu_index.py | 2 ++ diff --git a/recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch b/recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch new file mode 100644 index 00000000..9b82233c --- /dev/null +++ b/recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch @@ -0,0 +1,127 @@ +From 718d40c96dcfe142082472b2a3ca501945fe9abf Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Wed, 3 Mar 2021 23:59:39 +0100 +Subject: [PATCH 18/18] handle long vs. windows in swigfaiss.swig + +--- + faiss/python/swigfaiss.swig | 40 ++++++++++++++++++------------------- + tests/test_build_blocks.py | 4 +--- + 2 files changed, 21 insertions(+), 23 deletions(-) + +diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig +index 3113a136..3086ffab 100644 +--- a/faiss/python/swigfaiss.swig ++++ b/faiss/python/swigfaiss.swig +@@ -204,19 +204,19 @@ namespace std { + %template(DoubleVector) std::vector; + %template(ByteVector) std::vector; + %template(CharVector) std::vector; +-// NOTE(hoss): Using unsigned long instead of uint64_t because OSX defines +-// uint64_t as unsigned long long, which SWIG is not aware of. +-%template(Uint64Vector) std::vector; +-%template(LongVector) std::vector; +-%template(LongLongVector) std::vector; +-%template(IntVector) std::vector; +-%template(UInt32Vector) std::vector; + %template(Int16Vector) std::vector; ++// consider renaming to Int32Vector / Int64Vector ++%template(IntVector) std::vector; ++%template(LongVector) std::vector; ++// TODO: deprecate/remove LongLongVector ++%template(LongLongVector) std::vector; + %template(UInt16Vector) std::vector; ++%template(UInt32Vector) std::vector; ++%template(Uint64Vector) std::vector; + + %template(FloatVectorVector) std::vector >; + %template(ByteVectorVector) std::vector >; +-%template(LongVectorVector) std::vector >; ++%template(LongVectorVector) std::vector >; + %template(VectorTransformVector) std::vector; + %template(OperatingPointVector) std::vector; + %template(InvertedListsPtrVector) std::vector; +@@ -300,9 +300,9 @@ void gpu_sync_all_devices() + + %} + +-%template() std::pair; +-%template() std::map >; +-%template() std::map > >; ++%template() std::pair; ++%template() std::map >; ++%template() std::map > >; + + // causes weird wrapper bug + %ignore *::allocMemoryHandle; +@@ -317,12 +317,12 @@ typedef CUstream_st* cudaStream_t; + %inline %{ + + // interop between pytorch exposed cudaStream_t and faiss +-cudaStream_t cast_integer_to_cudastream_t(long long x) { ++cudaStream_t cast_integer_to_cudastream_t(int64_t x) { + return (cudaStream_t) x; + } + +-long long cast_cudastream_t_to_integer(cudaStream_t x) { +- return (long) x; ++int64_t cast_cudastream_t_to_integer(cudaStream_t x) { ++ return (int64_t) x; + } + + %} +@@ -898,23 +898,23 @@ void *memcpy(void *dest, const void *src, size_t n); + *******************************************************************/ + + %inline %{ +-uint8_t * cast_integer_to_uint8_ptr (long long x) { ++uint8_t * cast_integer_to_uint8_ptr (int64_t x) { + return (uint8_t*)x; + } + +-float * cast_integer_to_float_ptr (long long x) { ++float * cast_integer_to_float_ptr (int64_t x) { + return (float*)x; + } + +-faiss::Index::idx_t* cast_integer_to_idx_t_ptr (long long x) { ++faiss::Index::idx_t* cast_integer_to_idx_t_ptr (int64_t x) { + return (faiss::Index::idx_t*)x; + } + +-int * cast_integer_to_int_ptr (long long x) { ++int * cast_integer_to_int_ptr (int64_t x) { + return (int*)x; + } + +-void * cast_integer_to_void_ptr (long long x) { ++void * cast_integer_to_void_ptr (int64_t x) { + return (void*)x; + } + +@@ -941,7 +941,7 @@ struct MapLong2Long { + } + } + +- long search(int64_t key) { ++ int64_t search(int64_t key) { + if (map.count(key) == 0) { + return -1; + } else { +diff --git a/tests/test_build_blocks.py b/tests/test_build_blocks.py +index 713ed8fb..dd6b5961 100644 +--- a/tests/test_build_blocks.py ++++ b/tests/test_build_blocks.py +@@ -615,9 +615,7 @@ class TestSWIGWrap(unittest.TestCase): + def test_int64(self): + # see https://github.com/facebookresearch/faiss/issues/1529 + sizeof_long = array.array("l").itemsize +- if sizeof_long == 4: +- v = faiss.LongLongVector() +- elif sizeof_long == 8: ++ if sizeof_long in (4, 8): + v = faiss.LongVector() + else: + raise AssertionError("weird long size") +-- +2.29.2.windows.3 + From 90867d1593cf7ee0c8de9fa8431b42847d71b9fa Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 5 Mar 2021 09:56:37 +0100 Subject: [PATCH 05/12] add exploratory patch for replacing non-portable methods --- recipe/meta.yaml | 6 + recipe/patches/0001-use-c-14.patch | 2 +- ...ers-in-faiss-gpu-CMakeLists.txt-1666.patch | 2 +- ...003-update-util-guard-for-compute_86.patch | 2 +- ...missing-includes-for-std-min-std-max.patch | 2 +- ...-test-that-fails-without-GPU-drivers.patch | 2 +- ...-detection-platform-independent-1600.patch | 2 +- ...ke-setup.py-win-avx2-compatible-1682.patch | 2 +- ...essages-and-errors-in-loader.py-1683.patch | 2 +- ...ble-AVX2-switch-in-CMakeLists.txt-16.patch | 2 +- .../patches/0010-Win-AVX2-compat-1681.patch | 2 +- ...011-Windows-portable-intrinsics-1684.patch | 2 +- ...eader-target-directory-to-outputname.patch | 2 +- .../patches/0013-Fix-swig-vs.-AVX2-1717.patch | 2 +- ...ll-fixes-for-compilation-on-ARM-1655.patch | 2 +- ...erload-__builtin_ctz-for-win-cuda-11.patch | 2 +- ...fix-int32-vs-int64-issue-in-GPU-test.patch | 2 +- ...-skip-test_stress-for-GPU-on-windows.patch | 2 +- ...e-long-vs.-windows-in-swigfaiss.swig.patch | 2 +- ...rtable-Vector-names-and-replace-with.patch | 179 ++++++++++++++++++ 20 files changed, 203 insertions(+), 18 deletions(-) create mode 100644 recipe/patches/0019-deprecate-non-portable-Vector-names-and-replace-with.patch diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 3a317f28..5e82a194 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -106,6 +106,8 @@ source: - patches/0017-skip-test_stress-for-GPU-on-windows.patch # exploratory patch for long vs. windows in swigfaiss.swig - patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch + # exploratory patch for replacing non-portable methods + - patches/0019-deprecate-non-portable-Vector-names-and-replace-with.patch build: number: {{ number }} @@ -254,6 +256,10 @@ outputs: # this should have run without AVX2; skip for py36 due to NPY_DISABLE_CPU_FEATURES not working - python -c "q = open('log.txt').readlines(); import sys; sys.exit(0 if 'Successfully loaded faiss.' in [x[35:-1] for x in q] else 1)" # [py>36] + # temporary test for deprecations in patch 0019 + - python -c "import faiss; a = faiss.LongVector()" + - python -c "from faiss import *; a = LongVector()" + # running the following test requires an actual GPU device, which is not available in CI # - pytest faiss/gpu/test/ diff --git a/recipe/patches/0001-use-c-14.patch b/recipe/patches/0001-use-c-14.patch index 4ef1946b..55b94049 100644 --- a/recipe/patches/0001-use-c-14.patch +++ b/recipe/patches/0001-use-c-14.patch @@ -1,7 +1,7 @@ From 1b4e6f16de1bc6e6e7a104647625f45956356df5 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 26 Oct 2020 22:44:44 +0100 -Subject: [PATCH 01/18] use c++14 +Subject: [PATCH 01/19] use c++14 --- CMakeLists.txt | 2 +- diff --git a/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch b/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch index bd0f8f21..a77231b6 100644 --- a/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch +++ b/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch @@ -1,7 +1,7 @@ From e4b94c8a5ad0fc7632cf05006865dbd8feac2ed4 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Feb 2021 09:21:21 -0800 -Subject: [PATCH 02/18] Add missing headers in faiss/[gpu/]CMakeLists.txt +Subject: [PATCH 02/19] Add missing headers in faiss/[gpu/]CMakeLists.txt (#1666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 diff --git a/recipe/patches/0003-update-util-guard-for-compute_86.patch b/recipe/patches/0003-update-util-guard-for-compute_86.patch index cc89ac9f..b2da1360 100644 --- a/recipe/patches/0003-update-util-guard-for-compute_86.patch +++ b/recipe/patches/0003-update-util-guard-for-compute_86.patch @@ -1,7 +1,7 @@ From 84b98debfb92145a55e932aeb74eb0358ed15512 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 18:16:11 +0100 -Subject: [PATCH 03/18] update util-guard for compute_86 +Subject: [PATCH 03/19] update util-guard for compute_86 --- faiss/gpu/utils/DeviceDefs.cuh | 2 +- diff --git a/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch b/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch index 4db0e635..7f768f74 100644 --- a/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch +++ b/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch @@ -1,7 +1,7 @@ From c76bdcca995efee62144f0efbfdff1d7c7c3957c Mon Sep 17 00:00:00 2001 From: Lucas Hosseini Date: Thu, 7 Jan 2021 12:21:42 +0100 -Subject: [PATCH 04/18] Add missing includes for std::min/std::max. +Subject: [PATCH 04/19] Add missing includes for std::min/std::max. --- faiss/gpu/GpuIndex.cu | 1 + diff --git a/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch b/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch index 62fa33bf..6f7dc86a 100644 --- a/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch +++ b/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch @@ -1,7 +1,7 @@ From 1b4f509cdc91d38fecca58709093748324821931 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 18:36:12 +0100 -Subject: [PATCH 05/18] skip test that fails without GPU drivers +Subject: [PATCH 05/19] skip test that fails without GPU drivers --- tests/test_contrib.py | 3 +-- diff --git a/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch b/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch index f1e05dc3..cb49cb0e 100644 --- a/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch +++ b/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch @@ -1,7 +1,7 @@ From c329c59103f90a9b5fcec98f7a71f1bba4a3edbc Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 3 Feb 2021 08:00:39 -0800 -Subject: [PATCH 06/18] make AVX2-detection platform-independent (#1600) +Subject: [PATCH 06/19] make AVX2-detection platform-independent (#1600) Summary: In the context of https://github.com/conda-forge/faiss-split-feedstock/issues/23, I discussed with some of the conda-folks how we should support AVX2 (and potentially other builds) for faiss. In the meantime, we'd like to follow the model that faiss itself is using (i.e. build with AVX2 and without and then load the corresponding library at runtime depending on CPU capabilities). diff --git a/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch b/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch index 6b598d69..327381f4 100644 --- a/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch +++ b/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch @@ -1,7 +1,7 @@ From 4888d088f4bd62e82ddddd54fa98565ae201c566 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 15:07:28 -0800 -Subject: [PATCH 07/18] make setup.py win+avx2 compatible (#1682) +Subject: [PATCH 07/19] make setup.py win+avx2 compatible (#1682) Summary: While working on https://github.com/conda-forge/faiss-split-feedstock/pull/27, it turned out I needed diff --git a/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch b/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch index a488f3f6..dd749cd5 100644 --- a/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch +++ b/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch @@ -1,7 +1,7 @@ From af260092ea6fb23c61cea1d4b239da0334ad341c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 12 Feb 2021 10:57:44 -0800 -Subject: [PATCH 08/18] log success messages and errors in loader.py (#1683) +Subject: [PATCH 08/19] log success messages and errors in loader.py (#1683) Summary: This patch is to provide log information in case of failing to load the AVX2-enabled lib, diff --git a/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch b/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch index b562d0f1..f181b6ff 100644 --- a/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch +++ b/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch @@ -1,7 +1,7 @@ From 6b361d39aa12da3cc18b4c75317707bfa67dc1a1 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 14:05:11 -0800 -Subject: [PATCH 09/18] add msvc-compatible AVX2 switch in CMakeLists.txt +Subject: [PATCH 09/19] add msvc-compatible AVX2 switch in CMakeLists.txt (#1680) Summary: diff --git a/recipe/patches/0010-Win-AVX2-compat-1681.patch b/recipe/patches/0010-Win-AVX2-compat-1681.patch index 73b76c13..731e60f5 100644 --- a/recipe/patches/0010-Win-AVX2-compat-1681.patch +++ b/recipe/patches/0010-Win-AVX2-compat-1681.patch @@ -1,7 +1,7 @@ From 0e4bea421e47414853ff2345bc2438d22769424b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 08:04:30 -0800 -Subject: [PATCH 10/18] Win+AVX2 compat (#1681) +Subject: [PATCH 10/19] Win+AVX2 compat (#1681) Summary: Upstreaming patches from conda-forge/faiss-split-feedstock#27, follow-up (sorta) to https://github.com/facebookresearch/faiss/issues/1600. diff --git a/recipe/patches/0011-Windows-portable-intrinsics-1684.patch b/recipe/patches/0011-Windows-portable-intrinsics-1684.patch index 48c32386..1db28fab 100644 --- a/recipe/patches/0011-Windows-portable-intrinsics-1684.patch +++ b/recipe/patches/0011-Windows-portable-intrinsics-1684.patch @@ -1,7 +1,7 @@ From a987f2cf36de655be6273e7c91867c0dfedf09a9 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 Feb 2021 11:27:33 -0800 -Subject: [PATCH 11/18] Windows: portable intrinsics (#1684) +Subject: [PATCH 11/19] Windows: portable intrinsics (#1684) Summary: Trying to compile windows for AVX2 in https://github.com/conda-forge/faiss-split-feedstock/pull/27 diff --git a/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch b/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch index 295e6d5e..07b0cc5c 100644 --- a/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch +++ b/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch @@ -1,7 +1,7 @@ From 84072b99d78be9d94bf902e66fff396c1b94b21b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 14 Feb 2021 15:52:27 +0100 -Subject: [PATCH 12/18] adapt header target directory to outputname +Subject: [PATCH 12/19] adapt header target directory to outputname --- faiss/CMakeLists.txt | 11 ++++++++--- diff --git a/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch b/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch index c0de8967..26c2d40b 100644 --- a/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch +++ b/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch @@ -1,7 +1,7 @@ From 95e146d75efbbde1b55d5a9ae8f3957c6574ae5c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Mar 2021 03:43:07 -0800 -Subject: [PATCH 13/18] Fix swig vs. AVX2 (#1717) +Subject: [PATCH 13/19] Fix swig vs. AVX2 (#1717) Summary: Towards https://github.com/facebookresearch/faiss/issues/1711. There's still a test [error](https://github.com/facebookresearch/faiss/issues/1711#issuecomment-787509873) that won't be picked up unless an AVX2-specific CI run is added. diff --git a/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch b/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch index 86a80d3d..ed486098 100644 --- a/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch +++ b/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch @@ -1,7 +1,7 @@ From cab8a50faddf8413322c1646a8326e4c68f52405 Mon Sep 17 00:00:00 2001 From: Matthijs Douze Date: Fri, 29 Jan 2021 10:05:07 -0800 -Subject: [PATCH 14/18] Small fixes for compilation on ARM (#1655) +Subject: [PATCH 14/19] Small fixes for compilation on ARM (#1655) Summary: This PR fixes a few small issues with compilation on ARM. diff --git a/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch b/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch index c1df84c6..8566c0c0 100644 --- a/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch +++ b/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch @@ -1,7 +1,7 @@ From 1c0e08310b0ab7afb558f987adfdf3bbcca3085c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 Feb 2021 19:30:58 +0100 -Subject: [PATCH 15/18] do not overload __builtin_ctz for win + cuda>=11 +Subject: [PATCH 15/19] do not overload __builtin_ctz for win + cuda>=11 --- faiss/impl/platform_macros.h | 3 +++ diff --git a/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch b/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch index 3b32816c..77c5d20d 100644 --- a/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch +++ b/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch @@ -1,7 +1,7 @@ From 014f5b4ef37274cdbf295e336bea78c62d87fb7c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 19 Feb 2021 19:07:40 +0100 -Subject: [PATCH 16/18] fix int32 vs int64 issue in GPU test +Subject: [PATCH 16/19] fix int32 vs int64 issue in GPU test --- faiss/gpu/test/test_gpu_index.py | 7 +++++-- diff --git a/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch b/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch index 0264360c..6fbfb169 100644 --- a/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch +++ b/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch @@ -1,7 +1,7 @@ From eda6e7f7a743e7250162f03e5b3d825e65e3c167 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 19 Feb 2021 14:14:31 +0100 -Subject: [PATCH 17/18] skip test_stress for GPU on windows +Subject: [PATCH 17/19] skip test_stress for GPU on windows --- faiss/gpu/test/test_gpu_index.py | 2 ++ diff --git a/recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch b/recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch index 9b82233c..35a62af6 100644 --- a/recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch +++ b/recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch @@ -1,7 +1,7 @@ From 718d40c96dcfe142082472b2a3ca501945fe9abf Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 3 Mar 2021 23:59:39 +0100 -Subject: [PATCH 18/18] handle long vs. windows in swigfaiss.swig +Subject: [PATCH 18/19] handle long vs. windows in swigfaiss.swig --- faiss/python/swigfaiss.swig | 40 ++++++++++++++++++------------------- diff --git a/recipe/patches/0019-deprecate-non-portable-Vector-names-and-replace-with.patch b/recipe/patches/0019-deprecate-non-portable-Vector-names-and-replace-with.patch new file mode 100644 index 00000000..17ef3bcd --- /dev/null +++ b/recipe/patches/0019-deprecate-non-portable-Vector-names-and-replace-with.patch @@ -0,0 +1,179 @@ +From a187a6347382b157cfdfd10917094b89fe2d1c7e Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Fri, 5 Mar 2021 09:54:40 +0100 +Subject: [PATCH 19/19] deprecate non-portable Vector names and replace with + clearer nomenclature + +--- + faiss/python/__init__.py | 63 +++++++++++++++++++++++++++---------- + faiss/python/swigfaiss.swig | 21 ++++++------- + tests/test_build_blocks.py | 5 +-- + tests/test_oom_exception.py | 2 +- + 4 files changed, 58 insertions(+), 33 deletions(-) + +diff --git a/faiss/python/__init__.py b/faiss/python/__init__.py +index 315871aa..dbc1a8f2 100644 +--- a/faiss/python/__init__.py ++++ b/faiss/python/__init__.py +@@ -5,13 +5,14 @@ + + #@nolint + +-# not linting this file because it imports * form swigfaiss, which ++# not linting this file because it imports * from swigfaiss, which + # causes a ton of useless warnings. + + import numpy as np + import sys + import inspect + import array ++import warnings + + # We import * so that the symbol foo can be accessed as faiss.foo. + from .loader import * +@@ -523,7 +524,7 @@ def index_cpu_to_gpu_multiple_py(resources, index, co=None, gpus=None): + if gpus is None: + gpus = range(len(resources)) + vres = GpuResourcesVector() +- vdev = IntVector() ++ vdev = Int32Vector() + for i, res in zip(gpus, resources): + vdev.push_back(i) + vres.push_back(res) +@@ -635,23 +636,53 @@ def knn_gpu(res, xb, xq, k, D=None, I=None, metric=METRIC_L2): + # numpy array / std::vector conversions + ########################################### + +-# mapping from vector names in swigfaiss.swig and the numpy dtype names +-vector_name_map = { +- 'Float': 'float32', +- 'Byte': 'uint8', +- 'Char': 'int8', +- 'Uint64': 'uint64', +- 'LongLong': 'int64', +- 'Int': 'int32', +- 'Double': 'float64' ++sizeof_long = array.array('l').itemsize ++deprecated_name_map = { ++ 'Float': 'Float32', ++ 'Double': 'Float64', ++ 'Char': 'Int8', ++ 'Int': 'Int32', ++ 'Long': 'Int32' if sizeof_long == 4 else 'Int64', ++ 'LongLong': 'Int64', ++ 'Byte': 'UInt8' + } + +-sizeof_long = array.array('l').itemsize +-if sizeof_long == 4: +- vector_name_map["Long"] = 'int32' +-elif sizeof_long == 8: +- vector_name_map["Long"] = 'int64' ++def make_deprecated_class(deprecated_name, base_name): ++ """ ++ Dynamically construct deprecated classes and add warning to their __new__-method. ++ ++ We do this here because the classes are defined in the swig interface, ++ making it cumbersome to add the following functionality there directly. ++ """ ++ base_class = globals()[base_name] ++ def new_meth(cls, *args, **kwargs): ++ msg = f"The class faiss.{deprecated_name} is deprecated in favour of faiss.{base_name}!" ++ warnings.warn(msg, DeprecationWarning, stacklevel=2) ++ instance = super(base_class, cls).__new__(cls, *args, **kwargs) ++ return instance + ++ # three-argument version of type uses (name, tuple-of-bases, dict-of-attributes) ++ return type(deprecated_name, (base_class,), {"__new__": new_meth}) ++ ++for depr_prefix, base_prefix in deprecated_name_map.items(): ++ klazz = make_deprecated_class(depr_prefix + "Vector", base_prefix + "Vector") ++ globals()[depr_prefix + "Vector"] = klazz ++ ++# mapping from vector names in swigfaiss.swig and the numpy dtype names ++# TODO: once deprecated classes are removed, remove the dict and just use .lower() below ++vector_name_map = { ++ 'Float32': 'float32', ++ 'Float64': 'float64', ++ 'Int8': 'int8', ++ 'Int16': 'int16', ++ 'Int32': 'int32', ++ 'Int64': 'int64', ++ 'UInt8': 'uint8', ++ 'UInt16': 'uint16', ++ 'UInt32': 'uint32', ++ 'UInt64': 'uint64', ++ **{k: v.lower() for k, v in deprecated_name_map.items()} ++} + + + def vector_to_array(v): +diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig +index 3086ffab..1e7bdf3b 100644 +--- a/faiss/python/swigfaiss.swig ++++ b/faiss/python/swigfaiss.swig +@@ -200,23 +200,20 @@ namespace std { + %include + + // primitive array types +-%template(FloatVector) std::vector; +-%template(DoubleVector) std::vector; +-%template(ByteVector) std::vector; +-%template(CharVector) std::vector; ++%template(Float32Vector) std::vector; ++%template(Float64Vector) std::vector; ++%template(Int8Vector) std::vector; + %template(Int16Vector) std::vector; +-// consider renaming to Int32Vector / Int64Vector +-%template(IntVector) std::vector; +-%template(LongVector) std::vector; +-// TODO: deprecate/remove LongLongVector +-%template(LongLongVector) std::vector; ++%template(Int32Vector) std::vector; ++%template(Int64Vector) std::vector; ++%template(UInt8Vector) std::vector; + %template(UInt16Vector) std::vector; + %template(UInt32Vector) std::vector; + %template(Uint64Vector) std::vector; + +-%template(FloatVectorVector) std::vector >; +-%template(ByteVectorVector) std::vector >; +-%template(LongVectorVector) std::vector >; ++%template(Float32VectorVector) std::vector >; ++%template(UInt8VectorVector) std::vector >; ++%template(Int64VectorVector) std::vector >; + %template(VectorTransformVector) std::vector; + %template(OperatingPointVector) std::vector; + %template(InvertedListsPtrVector) std::vector; +diff --git a/tests/test_build_blocks.py b/tests/test_build_blocks.py +index dd6b5961..5534f39a 100644 +--- a/tests/test_build_blocks.py ++++ b/tests/test_build_blocks.py +@@ -615,10 +615,7 @@ class TestSWIGWrap(unittest.TestCase): + def test_int64(self): + # see https://github.com/facebookresearch/faiss/issues/1529 + sizeof_long = array.array("l").itemsize +- if sizeof_long in (4, 8): +- v = faiss.LongVector() +- else: +- raise AssertionError("weird long size") ++ v = faiss.Int64Vector() + + for i in range(10): + v.push_back(i) +diff --git a/tests/test_oom_exception.py b/tests/test_oom_exception.py +index 341004d9..e5300884 100644 +--- a/tests/test_oom_exception.py ++++ b/tests/test_oom_exception.py +@@ -19,7 +19,7 @@ class TestOOMException(unittest.TestCase): + # make sure that allocing more than 10G will fail + resource.setrlimit(resource.RLIMIT_AS, (10 * 1024 * 1024, hard_as)) + try: +- x = faiss.IntVector() ++ x = faiss.Int32Vector() + try: + x.resize(10**11) # 400 G of RAM + except MemoryError: +-- +2.29.2.windows.3 + From 4d7caf46c5f64e17922c7a521c51b3ebd9b89a34 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 5 Mar 2021 15:34:48 +0100 Subject: [PATCH 06/12] fix oversight in build-pkg.bat --- recipe/build-pkg.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/build-pkg.bat b/recipe/build-pkg.bat index 81f13dff..b373b50c 100644 --- a/recipe/build-pkg.bat +++ b/recipe/build-pkg.bat @@ -35,7 +35,7 @@ if %ERRORLEVEL% neq 0 exit 1 cmake -B _build_python_avx2 ^ -Dfaiss_ROOT=_libfaiss_avx2_stage ^ -DFAISS_OPT_LEVEL=avx2 ^ - -DFAISS_ENABLE_GPU=OFF ^ + -DFAISS_ENABLE_GPU=!FAISS_ENABLE_GPU! ^ -DCMAKE_BUILD_TYPE=Release ^ -DPython_EXECUTABLE="%PYTHON%" ^ faiss/python From 7ff43bb6292cdb7716214ecd1d10e7aa64c87b15 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 8 Mar 2021 20:36:57 +0100 Subject: [PATCH 07/12] update patches with some upstream work --- recipe/meta.yaml | 12 ++-- recipe/patches/0001-use-c-14.patch | 2 +- ...ers-in-faiss-gpu-CMakeLists.txt-1666.patch | 2 +- ...003-update-util-guard-for-compute_86.patch | 2 +- ...missing-includes-for-std-min-std-max.patch | 2 +- ...-test-that-fails-without-GPU-drivers.patch | 2 +- ...-detection-platform-independent-1600.patch | 2 +- ...ke-setup.py-win-avx2-compatible-1682.patch | 2 +- ...essages-and-errors-in-loader.py-1683.patch | 2 +- ...ble-AVX2-switch-in-CMakeLists.txt-16.patch | 2 +- .../patches/0010-Win-AVX2-compat-1681.patch | 2 +- ...011-Windows-portable-intrinsics-1684.patch | 2 +- ...eader-target-directory-to-outputname.patch | 2 +- .../patches/0013-Fix-swig-vs.-AVX2-1717.patch | 2 +- ...ll-fixes-for-compilation-on-ARM-1655.patch | 2 +- ...erload-__builtin_ctz-for-win-cuda-11.patch | 31 --------- ...5-roll-up-of-fixes-in-1737-1738-1739.patch | 64 +++++++++++++++++++ ...fix-int32-vs-int64-issue-in-GPU-test.patch | 30 --------- ...skip-test_stress-for-GPU-on-windows.patch} | 4 +- ...-long-vs.-windows-in-swigfaiss.swig.patch} | 4 +- ...table-Vector-names-and-replace-with.patch} | 4 +- 21 files changed, 89 insertions(+), 88 deletions(-) delete mode 100644 recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch create mode 100644 recipe/patches/0015-roll-up-of-fixes-in-1737-1738-1739.patch delete mode 100644 recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch rename recipe/patches/{0017-skip-test_stress-for-GPU-on-windows.patch => 0016-skip-test_stress-for-GPU-on-windows.patch} (86%) rename recipe/patches/{0018-handle-long-vs.-windows-in-swigfaiss.swig.patch => 0017-handle-long-vs.-windows-in-swigfaiss.swig.patch} (97%) rename recipe/patches/{0019-deprecate-non-portable-Vector-names-and-replace-with.patch => 0018-deprecate-non-portable-Vector-names-and-replace-with.patch} (98%) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 5e82a194..1576dae5 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -98,16 +98,14 @@ source: - patches/0013-Fix-swig-vs.-AVX2-1717.patch # backport fix for flaky test under AVX2; from faiss#1655, can be dropped for ver>1.7.0 - patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch - # avoid warning on win + cuda >= 11 - - patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch - # fix GPU test on windows - - patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch + # backport of facebookresearch/faiss#1737-1739 (squashed), can be dropped for ver>1.7.0 + - patches/0015-roll-up-of-fixes-in-1737-1738-1739.patch # patch for avoiding crash in GPU test suite on windows - - patches/0017-skip-test_stress-for-GPU-on-windows.patch + - patches/0016-skip-test_stress-for-GPU-on-windows.patch # exploratory patch for long vs. windows in swigfaiss.swig - - patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch + - patches/0017-handle-long-vs.-windows-in-swigfaiss.swig.patch # exploratory patch for replacing non-portable methods - - patches/0019-deprecate-non-portable-Vector-names-and-replace-with.patch + - patches/0018-deprecate-non-portable-Vector-names-and-replace-with.patch build: number: {{ number }} diff --git a/recipe/patches/0001-use-c-14.patch b/recipe/patches/0001-use-c-14.patch index 55b94049..4ef1946b 100644 --- a/recipe/patches/0001-use-c-14.patch +++ b/recipe/patches/0001-use-c-14.patch @@ -1,7 +1,7 @@ From 1b4e6f16de1bc6e6e7a104647625f45956356df5 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 26 Oct 2020 22:44:44 +0100 -Subject: [PATCH 01/19] use c++14 +Subject: [PATCH 01/18] use c++14 --- CMakeLists.txt | 2 +- diff --git a/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch b/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch index a77231b6..bd0f8f21 100644 --- a/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch +++ b/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch @@ -1,7 +1,7 @@ From e4b94c8a5ad0fc7632cf05006865dbd8feac2ed4 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Feb 2021 09:21:21 -0800 -Subject: [PATCH 02/19] Add missing headers in faiss/[gpu/]CMakeLists.txt +Subject: [PATCH 02/18] Add missing headers in faiss/[gpu/]CMakeLists.txt (#1666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 diff --git a/recipe/patches/0003-update-util-guard-for-compute_86.patch b/recipe/patches/0003-update-util-guard-for-compute_86.patch index b2da1360..cc89ac9f 100644 --- a/recipe/patches/0003-update-util-guard-for-compute_86.patch +++ b/recipe/patches/0003-update-util-guard-for-compute_86.patch @@ -1,7 +1,7 @@ From 84b98debfb92145a55e932aeb74eb0358ed15512 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 18:16:11 +0100 -Subject: [PATCH 03/19] update util-guard for compute_86 +Subject: [PATCH 03/18] update util-guard for compute_86 --- faiss/gpu/utils/DeviceDefs.cuh | 2 +- diff --git a/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch b/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch index 7f768f74..4db0e635 100644 --- a/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch +++ b/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch @@ -1,7 +1,7 @@ From c76bdcca995efee62144f0efbfdff1d7c7c3957c Mon Sep 17 00:00:00 2001 From: Lucas Hosseini Date: Thu, 7 Jan 2021 12:21:42 +0100 -Subject: [PATCH 04/19] Add missing includes for std::min/std::max. +Subject: [PATCH 04/18] Add missing includes for std::min/std::max. --- faiss/gpu/GpuIndex.cu | 1 + diff --git a/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch b/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch index 6f7dc86a..62fa33bf 100644 --- a/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch +++ b/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch @@ -1,7 +1,7 @@ From 1b4f509cdc91d38fecca58709093748324821931 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 18:36:12 +0100 -Subject: [PATCH 05/19] skip test that fails without GPU drivers +Subject: [PATCH 05/18] skip test that fails without GPU drivers --- tests/test_contrib.py | 3 +-- diff --git a/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch b/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch index cb49cb0e..f1e05dc3 100644 --- a/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch +++ b/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch @@ -1,7 +1,7 @@ From c329c59103f90a9b5fcec98f7a71f1bba4a3edbc Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 3 Feb 2021 08:00:39 -0800 -Subject: [PATCH 06/19] make AVX2-detection platform-independent (#1600) +Subject: [PATCH 06/18] make AVX2-detection platform-independent (#1600) Summary: In the context of https://github.com/conda-forge/faiss-split-feedstock/issues/23, I discussed with some of the conda-folks how we should support AVX2 (and potentially other builds) for faiss. In the meantime, we'd like to follow the model that faiss itself is using (i.e. build with AVX2 and without and then load the corresponding library at runtime depending on CPU capabilities). diff --git a/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch b/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch index 327381f4..6b598d69 100644 --- a/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch +++ b/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch @@ -1,7 +1,7 @@ From 4888d088f4bd62e82ddddd54fa98565ae201c566 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 15:07:28 -0800 -Subject: [PATCH 07/19] make setup.py win+avx2 compatible (#1682) +Subject: [PATCH 07/18] make setup.py win+avx2 compatible (#1682) Summary: While working on https://github.com/conda-forge/faiss-split-feedstock/pull/27, it turned out I needed diff --git a/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch b/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch index dd749cd5..a488f3f6 100644 --- a/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch +++ b/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch @@ -1,7 +1,7 @@ From af260092ea6fb23c61cea1d4b239da0334ad341c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 12 Feb 2021 10:57:44 -0800 -Subject: [PATCH 08/19] log success messages and errors in loader.py (#1683) +Subject: [PATCH 08/18] log success messages and errors in loader.py (#1683) Summary: This patch is to provide log information in case of failing to load the AVX2-enabled lib, diff --git a/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch b/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch index f181b6ff..b562d0f1 100644 --- a/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch +++ b/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch @@ -1,7 +1,7 @@ From 6b361d39aa12da3cc18b4c75317707bfa67dc1a1 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 14:05:11 -0800 -Subject: [PATCH 09/19] add msvc-compatible AVX2 switch in CMakeLists.txt +Subject: [PATCH 09/18] add msvc-compatible AVX2 switch in CMakeLists.txt (#1680) Summary: diff --git a/recipe/patches/0010-Win-AVX2-compat-1681.patch b/recipe/patches/0010-Win-AVX2-compat-1681.patch index 731e60f5..73b76c13 100644 --- a/recipe/patches/0010-Win-AVX2-compat-1681.patch +++ b/recipe/patches/0010-Win-AVX2-compat-1681.patch @@ -1,7 +1,7 @@ From 0e4bea421e47414853ff2345bc2438d22769424b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 08:04:30 -0800 -Subject: [PATCH 10/19] Win+AVX2 compat (#1681) +Subject: [PATCH 10/18] Win+AVX2 compat (#1681) Summary: Upstreaming patches from conda-forge/faiss-split-feedstock#27, follow-up (sorta) to https://github.com/facebookresearch/faiss/issues/1600. diff --git a/recipe/patches/0011-Windows-portable-intrinsics-1684.patch b/recipe/patches/0011-Windows-portable-intrinsics-1684.patch index 1db28fab..48c32386 100644 --- a/recipe/patches/0011-Windows-portable-intrinsics-1684.patch +++ b/recipe/patches/0011-Windows-portable-intrinsics-1684.patch @@ -1,7 +1,7 @@ From a987f2cf36de655be6273e7c91867c0dfedf09a9 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 Feb 2021 11:27:33 -0800 -Subject: [PATCH 11/19] Windows: portable intrinsics (#1684) +Subject: [PATCH 11/18] Windows: portable intrinsics (#1684) Summary: Trying to compile windows for AVX2 in https://github.com/conda-forge/faiss-split-feedstock/pull/27 diff --git a/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch b/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch index 07b0cc5c..295e6d5e 100644 --- a/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch +++ b/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch @@ -1,7 +1,7 @@ From 84072b99d78be9d94bf902e66fff396c1b94b21b Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 14 Feb 2021 15:52:27 +0100 -Subject: [PATCH 12/19] adapt header target directory to outputname +Subject: [PATCH 12/18] adapt header target directory to outputname --- faiss/CMakeLists.txt | 11 ++++++++--- diff --git a/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch b/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch index 26c2d40b..c0de8967 100644 --- a/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch +++ b/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch @@ -1,7 +1,7 @@ From 95e146d75efbbde1b55d5a9ae8f3957c6574ae5c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Mar 2021 03:43:07 -0800 -Subject: [PATCH 13/19] Fix swig vs. AVX2 (#1717) +Subject: [PATCH 13/18] Fix swig vs. AVX2 (#1717) Summary: Towards https://github.com/facebookresearch/faiss/issues/1711. There's still a test [error](https://github.com/facebookresearch/faiss/issues/1711#issuecomment-787509873) that won't be picked up unless an AVX2-specific CI run is added. diff --git a/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch b/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch index ed486098..86a80d3d 100644 --- a/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch +++ b/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch @@ -1,7 +1,7 @@ From cab8a50faddf8413322c1646a8326e4c68f52405 Mon Sep 17 00:00:00 2001 From: Matthijs Douze Date: Fri, 29 Jan 2021 10:05:07 -0800 -Subject: [PATCH 14/19] Small fixes for compilation on ARM (#1655) +Subject: [PATCH 14/18] Small fixes for compilation on ARM (#1655) Summary: This PR fixes a few small issues with compilation on ARM. diff --git a/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch b/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch deleted file mode 100644 index 8566c0c0..00000000 --- a/recipe/patches/0015-do-not-overload-__builtin_ctz-for-win-cuda-11.patch +++ /dev/null @@ -1,31 +0,0 @@ -From 1c0e08310b0ab7afb558f987adfdf3bbcca3085c Mon Sep 17 00:00:00 2001 -From: "H. Vetinari" -Date: Thu, 18 Feb 2021 19:30:58 +0100 -Subject: [PATCH 15/19] do not overload __builtin_ctz for win + cuda>=11 - ---- - faiss/impl/platform_macros.h | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/faiss/impl/platform_macros.h b/faiss/impl/platform_macros.h -index 1f4795c6..7efc9483 100644 ---- a/faiss/impl/platform_macros.h -+++ b/faiss/impl/platform_macros.h -@@ -39,11 +39,14 @@ inline int __builtin_ctzll(uint64_t x) { - return (int)ret; - } - -+// cudatoolkit provides __builtin_ctz for NVCC >= 11.0 -+#if ! defined(__CUDACC__) || __CUDACC_VER_MAJOR__ < 11 - inline int __builtin_ctz(unsigned long x) { - unsigned long ret; - _BitScanForward(&ret, x); - return (int)ret; - } -+#endif - - inline int __builtin_clzll(uint64_t x) { - return (int)__lzcnt64(x); --- -2.29.2.windows.3 - diff --git a/recipe/patches/0015-roll-up-of-fixes-in-1737-1738-1739.patch b/recipe/patches/0015-roll-up-of-fixes-in-1737-1738-1739.patch new file mode 100644 index 00000000..728444b5 --- /dev/null +++ b/recipe/patches/0015-roll-up-of-fixes-in-1737-1738-1739.patch @@ -0,0 +1,64 @@ +From 0cd0a82f9cc6b5e6a386671f74a1c32de9014eb7 Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Fri, 5 Mar 2021 13:41:44 -0800 +Subject: [PATCH 15/18] roll-up of fixes in #1737, #1738, #1739 + +--- + faiss/gpu/test/test_gpu_index.py | 7 +++++-- + faiss/impl/platform_macros.h | 3 +++ + tests/test_index_accuracy.py | 2 +- + 3 files changed, 9 insertions(+), 3 deletions(-) + +diff --git a/faiss/gpu/test/test_gpu_index.py b/faiss/gpu/test/test_gpu_index.py +index d96bd13a..d5af7a46 100755 +--- a/faiss/gpu/test/test_gpu_index.py ++++ b/faiss/gpu/test/test_gpu_index.py +@@ -61,8 +61,11 @@ class EvalIVFPQAccuracy(unittest.TestCase): + index.train(xt) + ts.append(time.time()) + +- # adding some ids because there was a bug in this case +- index.add_with_ids(xb, np.arange(nb) * 3 + 12345) ++ # adding some ids because there was a bug in this case; ++ # those need to be cast to idx_t(= int64_t), because ++ # on windows the numpy int default is int32 ++ ids = (np.arange(nb) * 3 + 12345).astype('int64') ++ index.add_with_ids(xb, ids) + ts.append(time.time()) + + index.nprobe = 4 +diff --git a/faiss/impl/platform_macros.h b/faiss/impl/platform_macros.h +index 1f4795c6..7efc9483 100644 +--- a/faiss/impl/platform_macros.h ++++ b/faiss/impl/platform_macros.h +@@ -39,11 +39,14 @@ inline int __builtin_ctzll(uint64_t x) { + return (int)ret; + } + ++// cudatoolkit provides __builtin_ctz for NVCC >= 11.0 ++#if ! defined(__CUDACC__) || __CUDACC_VER_MAJOR__ < 11 + inline int __builtin_ctz(unsigned long x) { + unsigned long ret; + _BitScanForward(&ret, x); + return (int)ret; + } ++#endif + + inline int __builtin_clzll(uint64_t x) { + return (int)__lzcnt64(x); +diff --git a/tests/test_index_accuracy.py b/tests/test_index_accuracy.py +index 5f3b557e..b5ee5138 100644 +--- a/tests/test_index_accuracy.py ++++ b/tests/test_index_accuracy.py +@@ -723,7 +723,7 @@ class TestRefine(unittest.TestCase): + # the original recall@100 + recall2 = (I2 == Iref[:, :1]).sum() + # print("recalls", recall1, recall2) +- self.assertEquals(recall1, recall2) ++ self.assertEqual(recall1, recall2) + + def test_IP(self): + self.do_test(faiss.METRIC_INNER_PRODUCT) +-- +2.29.2.windows.3 + diff --git a/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch b/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch deleted file mode 100644 index 77c5d20d..00000000 --- a/recipe/patches/0016-fix-int32-vs-int64-issue-in-GPU-test.patch +++ /dev/null @@ -1,30 +0,0 @@ -From 014f5b4ef37274cdbf295e336bea78c62d87fb7c Mon Sep 17 00:00:00 2001 -From: "H. Vetinari" -Date: Fri, 19 Feb 2021 19:07:40 +0100 -Subject: [PATCH 16/19] fix int32 vs int64 issue in GPU test - ---- - faiss/gpu/test/test_gpu_index.py | 7 +++++-- - 1 file changed, 5 insertions(+), 2 deletions(-) - -diff --git a/faiss/gpu/test/test_gpu_index.py b/faiss/gpu/test/test_gpu_index.py -index d96bd13a..d5af7a46 100755 ---- a/faiss/gpu/test/test_gpu_index.py -+++ b/faiss/gpu/test/test_gpu_index.py -@@ -61,8 +61,11 @@ class EvalIVFPQAccuracy(unittest.TestCase): - index.train(xt) - ts.append(time.time()) - -- # adding some ids because there was a bug in this case -- index.add_with_ids(xb, np.arange(nb) * 3 + 12345) -+ # adding some ids because there was a bug in this case; -+ # those need to be cast to idx_t(= int64_t), because -+ # on windows the numpy int default is int32 -+ ids = (np.arange(nb) * 3 + 12345).astype('int64') -+ index.add_with_ids(xb, ids) - ts.append(time.time()) - - index.nprobe = 4 --- -2.29.2.windows.3 - diff --git a/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch b/recipe/patches/0016-skip-test_stress-for-GPU-on-windows.patch similarity index 86% rename from recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch rename to recipe/patches/0016-skip-test_stress-for-GPU-on-windows.patch index 6fbfb169..214e7b72 100644 --- a/recipe/patches/0017-skip-test_stress-for-GPU-on-windows.patch +++ b/recipe/patches/0016-skip-test_stress-for-GPU-on-windows.patch @@ -1,7 +1,7 @@ -From eda6e7f7a743e7250162f03e5b3d825e65e3c167 Mon Sep 17 00:00:00 2001 +From 8e93a51907db47ba8be7feb3b18db876f199c6d1 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 19 Feb 2021 14:14:31 +0100 -Subject: [PATCH 17/19] skip test_stress for GPU on windows +Subject: [PATCH 16/18] skip test_stress for GPU on windows --- faiss/gpu/test/test_gpu_index.py | 2 ++ diff --git a/recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch b/recipe/patches/0017-handle-long-vs.-windows-in-swigfaiss.swig.patch similarity index 97% rename from recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch rename to recipe/patches/0017-handle-long-vs.-windows-in-swigfaiss.swig.patch index 35a62af6..2d1181d9 100644 --- a/recipe/patches/0018-handle-long-vs.-windows-in-swigfaiss.swig.patch +++ b/recipe/patches/0017-handle-long-vs.-windows-in-swigfaiss.swig.patch @@ -1,7 +1,7 @@ -From 718d40c96dcfe142082472b2a3ca501945fe9abf Mon Sep 17 00:00:00 2001 +From 11e223ada29dfa6002c1dda335c73957be07a463 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 3 Mar 2021 23:59:39 +0100 -Subject: [PATCH 18/19] handle long vs. windows in swigfaiss.swig +Subject: [PATCH 17/18] handle long vs. windows in swigfaiss.swig --- faiss/python/swigfaiss.swig | 40 ++++++++++++++++++------------------- diff --git a/recipe/patches/0019-deprecate-non-portable-Vector-names-and-replace-with.patch b/recipe/patches/0018-deprecate-non-portable-Vector-names-and-replace-with.patch similarity index 98% rename from recipe/patches/0019-deprecate-non-portable-Vector-names-and-replace-with.patch rename to recipe/patches/0018-deprecate-non-portable-Vector-names-and-replace-with.patch index 17ef3bcd..104d9b58 100644 --- a/recipe/patches/0019-deprecate-non-portable-Vector-names-and-replace-with.patch +++ b/recipe/patches/0018-deprecate-non-portable-Vector-names-and-replace-with.patch @@ -1,7 +1,7 @@ -From a187a6347382b157cfdfd10917094b89fe2d1c7e Mon Sep 17 00:00:00 2001 +From 348534a9950ccfc5da53fae04d95aceb344fc378 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 5 Mar 2021 09:54:40 +0100 -Subject: [PATCH 19/19] deprecate non-portable Vector names and replace with +Subject: [PATCH 18/18] deprecate non-portable Vector names and replace with clearer nomenclature --- From d7111ecf8a7339e844be06137ddbe3a18c01fbb6 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 9 Mar 2021 08:26:36 +0100 Subject: [PATCH 08/12] Revert "CMAKE_GENERATOR workaround for vs2019 from conda-forge" This reverts commit 66f583ec8e9b88cf52fe800c88fe8070fba7925b. --- recipe/build-lib.bat | 3 --- recipe/build-pkg.bat | 3 --- 2 files changed, 6 deletions(-) diff --git a/recipe/build-lib.bat b/recipe/build-lib.bat index 15ba3e83..62c3d6c3 100644 --- a/recipe/build-lib.bat +++ b/recipe/build-lib.bat @@ -45,9 +45,6 @@ if "%cuda_compiler_version%"=="None" ( echo Set up extra cmake-args: CUDA_CONFIG_ARGS=!CUDA_CONFIG_ARGS! ) -:: workaround for https://github.com/conda-forge/vc-feedstock/issues/21 -set "CMAKE_GENERATOR=Visual Studio 16 2019" - :: Build faiss.dll depending on $CF_FAISS_BUILD (either "generic" or "avx2") cmake -B _build_%CF_FAISS_BUILD% ^ -DBUILD_SHARED_LIBS=ON ^ diff --git a/recipe/build-pkg.bat b/recipe/build-pkg.bat index b373b50c..0a810342 100644 --- a/recipe/build-pkg.bat +++ b/recipe/build-pkg.bat @@ -16,9 +16,6 @@ if "%cuda_compiler_version%"=="None" ( set "CUDAToolkit_ROOT=%CUDA_PATH%" ) -:: workaround for https://github.com/conda-forge/vc-feedstock/issues/21 -set "CMAKE_GENERATOR=Visual Studio 16 2019" - :: Build vanilla version (no avx2), see build-lib.bat cmake -B _build_python_generic ^ -Dfaiss_ROOT=_libfaiss_generic_stage ^ From 2a5af95ce0ba1e85dacaaa79a8ffb06b2087f386 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 15 Mar 2021 19:05:47 +0100 Subject: [PATCH 09/12] reflect changes from #37 --- recipe/build-lib.bat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/recipe/build-lib.bat b/recipe/build-lib.bat index 62c3d6c3..9c968977 100644 --- a/recipe/build-lib.bat +++ b/recipe/build-lib.bat @@ -20,15 +20,15 @@ if "%cuda_compiler_version%"=="None" ( REM windows support start with cuda 10.0 REM %MY_VAR:~0,2% selects first two characters if "%cuda_compiler_version:~0,2%"=="10" ( - set "CMAKE_CUDA_ARCHS=35-virtual;50-virtual;52-virtual;60-virtual;61-virtual;70-virtual;75-virtual;75-real" + set "CMAKE_CUDA_ARCHS=35-real;50-real;52-real;60-real;61-real;70-real;75" ) if "%cuda_compiler_version:~0,2%"=="11" ( if "%cuda_compiler_version:~0,4%"=="11.0" ( REM cuda 11.0 deprecates arches 35, 50 - set "CMAKE_CUDA_ARCHS=52-virtual;60-virtual;61-virtual;70-virtual;75-virtual;80-virtual;80-real" + set "CMAKE_CUDA_ARCHS=52-real;60-real;61-real;70-real;75-real;80" ) else ( REM cuda>=11.1 adds arch 86 - set "CMAKE_CUDA_ARCHS=52-virtual;60-virtual;61-virtual;70-virtual;75-virtual;80-virtual;86-virtual;86-real" + set "CMAKE_CUDA_ARCHS=52-real;60-real;61-real;70-real;75-real;80-real;86" ) ) From 6212efb9be08c26ff6a07e497e99871e92e0758c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 15 Mar 2021 19:06:16 +0100 Subject: [PATCH 10/12] bump build number --- recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 1576dae5..c97af144 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set version = "1.7.0" %} -{% set number = 5 %} +{% set number = 6 %} # see github.com/conda-forge/conda-forge.github.io/issues/1059 for naming discussion {% set faiss_proc_type = "cuda" if cuda_compiler_version != "None" else "cpu" %} From 21f46178a1222cf6f9eb3a3ff166b517b8ededbd Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 6 Apr 2021 18:22:13 +0200 Subject: [PATCH 11/12] use upstreamed patch Also remove one unconditional test suite skip for windows (patch 0005); replaced by filter condition in pytest invocation --- recipe/meta.yaml | 42 ++- recipe/patches/0001-use-c-14.patch | 2 +- ...ers-in-faiss-gpu-CMakeLists.txt-1666.patch | 2 +- ...003-update-util-guard-for-compute_86.patch | 2 +- ...missing-includes-for-std-min-std-max.patch | 2 +- ...detection-platform-independent-1600.patch} | 4 +- ...-test-that-fails-without-GPU-drivers.patch | 26 -- ...e-setup.py-win-avx2-compatible-1682.patch} | 4 +- ...ssages-and-errors-in-loader.py-1683.patch} | 4 +- ...le-AVX2-switch-in-CMakeLists.txt-16.patch} | 4 +- ....patch => 0009-Win-AVX2-compat-1681.patch} | 4 +- ...10-Windows-portable-intrinsics-1684.patch} | 4 +- ...atch => 0011-Fix-swig-vs.-AVX2-1717.patch} | 4 +- ...l-fixes-for-compilation-on-ARM-1655.patch} | 4 +- ...-roll-up-of-fixes-in-1737-1738-1739.patch} | 4 +- .../0014-Portable-SWIG-Vectors-1742.patch | 312 ++++++++++++++++++ ...ader-target-directory-to-outputname.patch} | 4 +- ...-skip-test_stress-for-GPU-on-windows.patch | 4 +- ...e-long-vs.-windows-in-swigfaiss.swig.patch | 127 ------- ...rtable-Vector-names-and-replace-with.patch | 179 ---------- 20 files changed, 357 insertions(+), 381 deletions(-) rename recipe/patches/{0006-make-AVX2-detection-platform-independent-1600.patch => 0005-make-AVX2-detection-platform-independent-1600.patch} (96%) delete mode 100644 recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch rename recipe/patches/{0007-make-setup.py-win-avx2-compatible-1682.patch => 0006-make-setup.py-win-avx2-compatible-1682.patch} (95%) rename recipe/patches/{0008-log-success-messages-and-errors-in-loader.py-1683.patch => 0007-log-success-messages-and-errors-in-loader.py-1683.patch} (94%) rename recipe/patches/{0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch => 0008-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch} (94%) rename recipe/patches/{0010-Win-AVX2-compat-1681.patch => 0009-Win-AVX2-compat-1681.patch} (96%) rename recipe/patches/{0011-Windows-portable-intrinsics-1684.patch => 0010-Windows-portable-intrinsics-1684.patch} (99%) rename recipe/patches/{0013-Fix-swig-vs.-AVX2-1717.patch => 0011-Fix-swig-vs.-AVX2-1717.patch} (96%) rename recipe/patches/{0014-Small-fixes-for-compilation-on-ARM-1655.patch => 0012-Small-fixes-for-compilation-on-ARM-1655.patch} (94%) rename recipe/patches/{0015-roll-up-of-fixes-in-1737-1738-1739.patch => 0013-roll-up-of-fixes-in-1737-1738-1739.patch} (94%) create mode 100644 recipe/patches/0014-Portable-SWIG-Vectors-1742.patch rename recipe/patches/{0012-adapt-header-target-directory-to-outputname.patch => 0015-adapt-header-target-directory-to-outputname.patch} (95%) delete mode 100644 recipe/patches/0017-handle-long-vs.-windows-in-swigfaiss.swig.patch delete mode 100644 recipe/patches/0018-deprecate-non-portable-Vector-names-and-replace-with.patch diff --git a/recipe/meta.yaml b/recipe/meta.yaml index c97af144..4f254685 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -78,34 +78,30 @@ source: - patches/0003-update-util-guard-for-compute_86.patch # single commit from facebookresearch/faiss#1610 - patches/0004-Add-missing-includes-for-std-min-std-max.patch - # skip test that fails without GPU drivers on windows - - patches/0005-skip-test-that-fails-without-GPU-drivers.patch # [win] # backport of facebookresearch/faiss#1600, can be dropped for ver>1.7.0 - - patches/0006-make-AVX2-detection-platform-independent-1600.patch + - patches/0005-make-AVX2-detection-platform-independent-1600.patch # backport of facebookresearch/faiss#1682, can be dropped for ver>1.7.0 - - patches/0007-make-setup.py-win-avx2-compatible-1682.patch + - patches/0006-make-setup.py-win-avx2-compatible-1682.patch # backport of facebookresearch/faiss#1683, can be dropped for ver>1.7.0 - - patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch + - patches/0007-log-success-messages-and-errors-in-loader.py-1683.patch # backport of facebookresearch/faiss#1680, can be dropped for ver>1.7.0 - - patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch + - patches/0008-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch # backport of facebookresearch/faiss#1681, can be dropped for ver>1.7.0 - - patches/0010-Win-AVX2-compat-1681.patch + - patches/0009-Win-AVX2-compat-1681.patch # backport of facebookresearch/faiss#1684, can be dropped for ver>1.7.0 - - patches/0011-Windows-portable-intrinsics-1684.patch - # adapt header target directory for faiss_avx2 - - patches/0012-adapt-header-target-directory-to-outputname.patch + - patches/0010-Windows-portable-intrinsics-1684.patch # backport of facebookresearch/faiss#1717, can be dropped for ver>1.7.0 - - patches/0013-Fix-swig-vs.-AVX2-1717.patch + - patches/0011-Fix-swig-vs.-AVX2-1717.patch # backport fix for flaky test under AVX2; from faiss#1655, can be dropped for ver>1.7.0 - - patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch + - patches/0012-Small-fixes-for-compilation-on-ARM-1655.patch # backport of facebookresearch/faiss#1737-1739 (squashed), can be dropped for ver>1.7.0 - - patches/0015-roll-up-of-fixes-in-1737-1738-1739.patch + - patches/0013-roll-up-of-fixes-in-1737-1738-1739.patch + # backport of facebookresearch/faiss#1742, can be dropped for ver>1.7.0 + - patches/0014-Portable-SWIG-Vectors-1742.patch + # adapt header target directory for faiss_avx2 + - patches/0015-adapt-header-target-directory-to-outputname.patch # patch for avoiding crash in GPU test suite on windows - patches/0016-skip-test_stress-for-GPU-on-windows.patch - # exploratory patch for long vs. windows in swigfaiss.swig - - patches/0017-handle-long-vs.-windows-in-swigfaiss.swig.patch - # exploratory patch for replacing non-portable methods - - patches/0018-deprecate-non-portable-Vector-names-and-replace-with.patch build: number: {{ number }} @@ -231,7 +227,10 @@ outputs: # we expect faiss will load the library with AVX2-support, see # https://github.com/facebookresearch/faiss/blob/master/faiss/python/loader.py#L48-L60 - python -c "from numpy.core._multiarray_umath import __cpu_features__; print(f'Testing version with AVX2-support - ' + str(__cpu_features__['AVX2']))" - - pytest tests --log-file-level=INFO --log-file=log.txt + - pytest tests --log-file-level=INFO --log-file=log.txt # [not win] + # TestComputeGT switches between CPU & GPU implementation depending on availability; + # GPU detection on windows is currently broken in CUDA, and would therefore break the test suite + - pytest tests --log-file-level=INFO --log-file=log.txt -k "not TestComputeGT" # [win] # print logfile for completeness (sleep so log has time to print) - cat log.txt && sleep 2 # [not win] - type log.txt # [win] @@ -248,16 +247,13 @@ outputs: {% endif %} - python -c "from numpy.core._multiarray_umath import __cpu_features__; print(f'Testing version with AVX2-support - ' + str(__cpu_features__['AVX2']))" # rerun test suite again without AVX2 support - - pytest tests --log-file-level=INFO --log-file=log.txt + - pytest tests --log-file-level=INFO --log-file=log.txt # [not win] + - pytest tests --log-file-level=INFO --log-file=log.txt -k "not TestComputeGT" # [win] - cat log.txt && sleep 2 # [not win] - type log.txt # [win] # this should have run without AVX2; skip for py36 due to NPY_DISABLE_CPU_FEATURES not working - python -c "q = open('log.txt').readlines(); import sys; sys.exit(0 if 'Successfully loaded faiss.' in [x[35:-1] for x in q] else 1)" # [py>36] - # temporary test for deprecations in patch 0019 - - python -c "import faiss; a = faiss.LongVector()" - - python -c "from faiss import *; a = LongVector()" - # running the following test requires an actual GPU device, which is not available in CI # - pytest faiss/gpu/test/ diff --git a/recipe/patches/0001-use-c-14.patch b/recipe/patches/0001-use-c-14.patch index 4ef1946b..74730ec7 100644 --- a/recipe/patches/0001-use-c-14.patch +++ b/recipe/patches/0001-use-c-14.patch @@ -1,7 +1,7 @@ From 1b4e6f16de1bc6e6e7a104647625f45956356df5 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 26 Oct 2020 22:44:44 +0100 -Subject: [PATCH 01/18] use c++14 +Subject: [PATCH 01/16] use c++14 --- CMakeLists.txt | 2 +- diff --git a/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch b/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch index bd0f8f21..3d34073f 100644 --- a/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch +++ b/recipe/patches/0002-Add-missing-headers-in-faiss-gpu-CMakeLists.txt-1666.patch @@ -1,7 +1,7 @@ From e4b94c8a5ad0fc7632cf05006865dbd8feac2ed4 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Feb 2021 09:21:21 -0800 -Subject: [PATCH 02/18] Add missing headers in faiss/[gpu/]CMakeLists.txt +Subject: [PATCH 02/16] Add missing headers in faiss/[gpu/]CMakeLists.txt (#1666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 diff --git a/recipe/patches/0003-update-util-guard-for-compute_86.patch b/recipe/patches/0003-update-util-guard-for-compute_86.patch index cc89ac9f..5c852479 100644 --- a/recipe/patches/0003-update-util-guard-for-compute_86.patch +++ b/recipe/patches/0003-update-util-guard-for-compute_86.patch @@ -1,7 +1,7 @@ From 84b98debfb92145a55e932aeb74eb0358ed15512 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 18:16:11 +0100 -Subject: [PATCH 03/18] update util-guard for compute_86 +Subject: [PATCH 03/16] update util-guard for compute_86 --- faiss/gpu/utils/DeviceDefs.cuh | 2 +- diff --git a/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch b/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch index 4db0e635..83669695 100644 --- a/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch +++ b/recipe/patches/0004-Add-missing-includes-for-std-min-std-max.patch @@ -1,7 +1,7 @@ From c76bdcca995efee62144f0efbfdff1d7c7c3957c Mon Sep 17 00:00:00 2001 From: Lucas Hosseini Date: Thu, 7 Jan 2021 12:21:42 +0100 -Subject: [PATCH 04/18] Add missing includes for std::min/std::max. +Subject: [PATCH 04/16] Add missing includes for std::min/std::max. --- faiss/gpu/GpuIndex.cu | 1 + diff --git a/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch b/recipe/patches/0005-make-AVX2-detection-platform-independent-1600.patch similarity index 96% rename from recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch rename to recipe/patches/0005-make-AVX2-detection-platform-independent-1600.patch index f1e05dc3..b40c5034 100644 --- a/recipe/patches/0006-make-AVX2-detection-platform-independent-1600.patch +++ b/recipe/patches/0005-make-AVX2-detection-platform-independent-1600.patch @@ -1,7 +1,7 @@ -From c329c59103f90a9b5fcec98f7a71f1bba4a3edbc Mon Sep 17 00:00:00 2001 +From f6d53dca943a64a614fcf30dfb8b6a5f92492365 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 3 Feb 2021 08:00:39 -0800 -Subject: [PATCH 06/18] make AVX2-detection platform-independent (#1600) +Subject: [PATCH 05/16] make AVX2-detection platform-independent (#1600) Summary: In the context of https://github.com/conda-forge/faiss-split-feedstock/issues/23, I discussed with some of the conda-folks how we should support AVX2 (and potentially other builds) for faiss. In the meantime, we'd like to follow the model that faiss itself is using (i.e. build with AVX2 and without and then load the corresponding library at runtime depending on CPU capabilities). diff --git a/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch b/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch deleted file mode 100644 index 62fa33bf..00000000 --- a/recipe/patches/0005-skip-test-that-fails-without-GPU-drivers.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 1b4f509cdc91d38fecca58709093748324821931 Mon Sep 17 00:00:00 2001 -From: "H. Vetinari" -Date: Wed, 17 Feb 2021 18:36:12 +0100 -Subject: [PATCH 05/18] skip test that fails without GPU drivers - ---- - tests/test_contrib.py | 3 +-- - 1 file changed, 1 insertion(+), 2 deletions(-) - -diff --git a/tests/test_contrib.py b/tests/test_contrib.py -index 85990813..65811c4d 100644 ---- a/tests/test_contrib.py -+++ b/tests/test_contrib.py -@@ -22,8 +22,7 @@ except: - - - --@unittest.skipIf(platform.python_version_tuple()[0] < '3', \ -- 'Submodule import broken in python 2.') -+@unittest.skipIf(True, 'No CUDA drivers in windows CI') - class TestComputeGT(unittest.TestCase): - - def test_compute_GT(self): --- -2.29.2.windows.3 - diff --git a/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch b/recipe/patches/0006-make-setup.py-win-avx2-compatible-1682.patch similarity index 95% rename from recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch rename to recipe/patches/0006-make-setup.py-win-avx2-compatible-1682.patch index 6b598d69..62ffbffe 100644 --- a/recipe/patches/0007-make-setup.py-win-avx2-compatible-1682.patch +++ b/recipe/patches/0006-make-setup.py-win-avx2-compatible-1682.patch @@ -1,7 +1,7 @@ -From 4888d088f4bd62e82ddddd54fa98565ae201c566 Mon Sep 17 00:00:00 2001 +From 281592dedcccf9cb86f0f60ee2272671300329a0 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 15:07:28 -0800 -Subject: [PATCH 07/18] make setup.py win+avx2 compatible (#1682) +Subject: [PATCH 06/16] make setup.py win+avx2 compatible (#1682) Summary: While working on https://github.com/conda-forge/faiss-split-feedstock/pull/27, it turned out I needed diff --git a/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch b/recipe/patches/0007-log-success-messages-and-errors-in-loader.py-1683.patch similarity index 94% rename from recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch rename to recipe/patches/0007-log-success-messages-and-errors-in-loader.py-1683.patch index a488f3f6..16b1037c 100644 --- a/recipe/patches/0008-log-success-messages-and-errors-in-loader.py-1683.patch +++ b/recipe/patches/0007-log-success-messages-and-errors-in-loader.py-1683.patch @@ -1,7 +1,7 @@ -From af260092ea6fb23c61cea1d4b239da0334ad341c Mon Sep 17 00:00:00 2001 +From dd8e2e88a498e5a95296780931836750d64dfc8c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 12 Feb 2021 10:57:44 -0800 -Subject: [PATCH 08/18] log success messages and errors in loader.py (#1683) +Subject: [PATCH 07/16] log success messages and errors in loader.py (#1683) Summary: This patch is to provide log information in case of failing to load the AVX2-enabled lib, diff --git a/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch b/recipe/patches/0008-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch similarity index 94% rename from recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch rename to recipe/patches/0008-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch index b562d0f1..1b06c65d 100644 --- a/recipe/patches/0009-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch +++ b/recipe/patches/0008-add-msvc-compatible-AVX2-switch-in-CMakeLists.txt-16.patch @@ -1,7 +1,7 @@ -From 6b361d39aa12da3cc18b4c75317707bfa67dc1a1 Mon Sep 17 00:00:00 2001 +From 2c8c67428c2d9a56c0bc82669ed7e326a862564d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 14:05:11 -0800 -Subject: [PATCH 09/18] add msvc-compatible AVX2 switch in CMakeLists.txt +Subject: [PATCH 08/16] add msvc-compatible AVX2 switch in CMakeLists.txt (#1680) Summary: diff --git a/recipe/patches/0010-Win-AVX2-compat-1681.patch b/recipe/patches/0009-Win-AVX2-compat-1681.patch similarity index 96% rename from recipe/patches/0010-Win-AVX2-compat-1681.patch rename to recipe/patches/0009-Win-AVX2-compat-1681.patch index 73b76c13..fe048cb8 100644 --- a/recipe/patches/0010-Win-AVX2-compat-1681.patch +++ b/recipe/patches/0009-Win-AVX2-compat-1681.patch @@ -1,7 +1,7 @@ -From 0e4bea421e47414853ff2345bc2438d22769424b Mon Sep 17 00:00:00 2001 +From b2f882974695ad90293585e6bbed3dc7d29d351d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 17 Feb 2021 08:04:30 -0800 -Subject: [PATCH 10/18] Win+AVX2 compat (#1681) +Subject: [PATCH 09/16] Win+AVX2 compat (#1681) Summary: Upstreaming patches from conda-forge/faiss-split-feedstock#27, follow-up (sorta) to https://github.com/facebookresearch/faiss/issues/1600. diff --git a/recipe/patches/0011-Windows-portable-intrinsics-1684.patch b/recipe/patches/0010-Windows-portable-intrinsics-1684.patch similarity index 99% rename from recipe/patches/0011-Windows-portable-intrinsics-1684.patch rename to recipe/patches/0010-Windows-portable-intrinsics-1684.patch index 48c32386..3e441957 100644 --- a/recipe/patches/0011-Windows-portable-intrinsics-1684.patch +++ b/recipe/patches/0010-Windows-portable-intrinsics-1684.patch @@ -1,7 +1,7 @@ -From a987f2cf36de655be6273e7c91867c0dfedf09a9 Mon Sep 17 00:00:00 2001 +From 4b1ca43de9c86ca487d86d5259903b7729160802 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 18 Feb 2021 11:27:33 -0800 -Subject: [PATCH 11/18] Windows: portable intrinsics (#1684) +Subject: [PATCH 10/16] Windows: portable intrinsics (#1684) Summary: Trying to compile windows for AVX2 in https://github.com/conda-forge/faiss-split-feedstock/pull/27 diff --git a/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch b/recipe/patches/0011-Fix-swig-vs.-AVX2-1717.patch similarity index 96% rename from recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch rename to recipe/patches/0011-Fix-swig-vs.-AVX2-1717.patch index c0de8967..a60ceb76 100644 --- a/recipe/patches/0013-Fix-swig-vs.-AVX2-1717.patch +++ b/recipe/patches/0011-Fix-swig-vs.-AVX2-1717.patch @@ -1,7 +1,7 @@ -From 95e146d75efbbde1b55d5a9ae8f3957c6574ae5c Mon Sep 17 00:00:00 2001 +From 005a0ef4a2c5375190c1059ff52537b8a0fcc4cd Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 4 Mar 2021 03:43:07 -0800 -Subject: [PATCH 13/18] Fix swig vs. AVX2 (#1717) +Subject: [PATCH 11/16] Fix swig vs. AVX2 (#1717) Summary: Towards https://github.com/facebookresearch/faiss/issues/1711. There's still a test [error](https://github.com/facebookresearch/faiss/issues/1711#issuecomment-787509873) that won't be picked up unless an AVX2-specific CI run is added. diff --git a/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch b/recipe/patches/0012-Small-fixes-for-compilation-on-ARM-1655.patch similarity index 94% rename from recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch rename to recipe/patches/0012-Small-fixes-for-compilation-on-ARM-1655.patch index 86a80d3d..df733fb9 100644 --- a/recipe/patches/0014-Small-fixes-for-compilation-on-ARM-1655.patch +++ b/recipe/patches/0012-Small-fixes-for-compilation-on-ARM-1655.patch @@ -1,7 +1,7 @@ -From cab8a50faddf8413322c1646a8326e4c68f52405 Mon Sep 17 00:00:00 2001 +From 578518a3e9e683c0dc3a7831055a3290330be813 Mon Sep 17 00:00:00 2001 From: Matthijs Douze Date: Fri, 29 Jan 2021 10:05:07 -0800 -Subject: [PATCH 14/18] Small fixes for compilation on ARM (#1655) +Subject: [PATCH 12/16] Small fixes for compilation on ARM (#1655) Summary: This PR fixes a few small issues with compilation on ARM. diff --git a/recipe/patches/0015-roll-up-of-fixes-in-1737-1738-1739.patch b/recipe/patches/0013-roll-up-of-fixes-in-1737-1738-1739.patch similarity index 94% rename from recipe/patches/0015-roll-up-of-fixes-in-1737-1738-1739.patch rename to recipe/patches/0013-roll-up-of-fixes-in-1737-1738-1739.patch index 728444b5..fa394124 100644 --- a/recipe/patches/0015-roll-up-of-fixes-in-1737-1738-1739.patch +++ b/recipe/patches/0013-roll-up-of-fixes-in-1737-1738-1739.patch @@ -1,7 +1,7 @@ -From 0cd0a82f9cc6b5e6a386671f74a1c32de9014eb7 Mon Sep 17 00:00:00 2001 +From ea40a7e02262d9b00242520487bcd06d69236d5f Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 5 Mar 2021 13:41:44 -0800 -Subject: [PATCH 15/18] roll-up of fixes in #1737, #1738, #1739 +Subject: [PATCH 13/16] roll-up of fixes in #1737, #1738, #1739 --- faiss/gpu/test/test_gpu_index.py | 7 +++++-- diff --git a/recipe/patches/0014-Portable-SWIG-Vectors-1742.patch b/recipe/patches/0014-Portable-SWIG-Vectors-1742.patch new file mode 100644 index 00000000..ceace7d4 --- /dev/null +++ b/recipe/patches/0014-Portable-SWIG-Vectors-1742.patch @@ -0,0 +1,312 @@ +From 592bb45193a44fefc722cbbeb58fe441c926c378 Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Tue, 6 Apr 2021 18:00:07 +0200 +Subject: [PATCH 14/16] Portable SWIG Vectors (#1742) + +Summary: +After initial positive feedback to the idea in https://github.com/facebookresearch/faiss/issues/1741 from mdouze, here are the patches +I currently have as a basis for discussion. + +Matthijs suggests to not bother with the deprecation warnings at all, which is fine for me +as well, though I would normally still advocate to provide users with _some_ advance notice +before removing parts of an interface. + +Fixes https://github.com/facebookresearch/faiss/issues/1741 + +PS. The deprecation warning is only shown once per session (per class) +PPS. I have tested in https://github.com/conda-forge/faiss-split-feedstock/pull/32 that the respective +classes remain available both through `import faiss` and `from faiss import *`. + +Pull Request resolved: https://github.com/facebookresearch/faiss/pull/1742 + +Reviewed By: mdouze + +Differential Revision: D26978886 + +Pulled By: beauby + +fbshipit-source-id: b52e2b5b5b0117af7cd95ef5df3128e9914633ad +--- + faiss/python/__init__.py | 94 ++++++++++++++++++++++++++++++------- + faiss/python/swigfaiss.swig | 50 ++++++++++---------- + tests/test_build_blocks.py | 8 +--- + tests/test_oom_exception.py | 2 +- + 4 files changed, 104 insertions(+), 50 deletions(-) + +diff --git a/faiss/python/__init__.py b/faiss/python/__init__.py +index 315871aa..235c129a 100644 +--- a/faiss/python/__init__.py ++++ b/faiss/python/__init__.py +@@ -5,13 +5,14 @@ + + #@nolint + +-# not linting this file because it imports * form swigfaiss, which ++# not linting this file because it imports * from swigfaiss, which + # causes a ton of useless warnings. + + import numpy as np + import sys + import inspect + import array ++import warnings + + # We import * so that the symbol foo can be accessed as faiss.foo. + from .loader import * +@@ -431,6 +432,47 @@ for symbol in dir(this_module): + if issubclass(the_class, ParameterSpace): + handle_ParameterSpace(the_class) + ++########################################### ++# Utility to add a deprecation warning to ++# classes from the SWIG interface ++########################################### ++ ++def _make_deprecated_swig_class(deprecated_name, base_name): ++ """ ++ Dynamically construct deprecated classes as wrappers around renamed ones ++ ++ The deprecation warning added in their __new__-method will trigger upon ++ construction of an instance of the class, but only once per session. ++ ++ We do this here (in __init__.py) because the base classes are defined in ++ the SWIG interface, making it cumbersome to add the deprecation there. ++ ++ Parameters ++ ---------- ++ deprecated_name : string ++ Name of the class to be deprecated; _not_ present in SWIG interface. ++ base_name : string ++ Name of the class that is replacing deprecated_name; must already be ++ imported into the current namespace. ++ ++ Returns ++ ------- ++ None ++ However, the deprecated class gets added to the faiss namespace ++ """ ++ base_class = globals()[base_name] ++ def new_meth(cls, *args, **kwargs): ++ msg = f"The class faiss.{deprecated_name} is deprecated in favour of faiss.{base_name}!" ++ warnings.warn(msg, DeprecationWarning, stacklevel=2) ++ instance = super(base_class, cls).__new__(cls, *args, **kwargs) ++ return instance ++ ++ # three-argument version of "type" uses (name, tuple-of-bases, dict-of-attributes) ++ klazz = type(deprecated_name, (base_class,), {"__new__": new_meth}) ++ ++ # this ends up adding the class to the "faiss" namespace, in a way that it ++ # is available both through "import faiss" and "from faiss import *" ++ globals()[deprecated_name] = klazz + + ########################################### + # Add Python references to objects +@@ -523,7 +565,7 @@ def index_cpu_to_gpu_multiple_py(resources, index, co=None, gpus=None): + if gpus is None: + gpus = range(len(resources)) + vres = GpuResourcesVector() +- vdev = IntVector() ++ vdev = Int32Vector() + for i, res in zip(gpus, resources): + vdev.push_back(i) + vres.push_back(res) +@@ -635,23 +677,43 @@ def knn_gpu(res, xb, xq, k, D=None, I=None, metric=METRIC_L2): + # numpy array / std::vector conversions + ########################################### + +-# mapping from vector names in swigfaiss.swig and the numpy dtype names +-vector_name_map = { +- 'Float': 'float32', +- 'Byte': 'uint8', +- 'Char': 'int8', +- 'Uint64': 'uint64', +- 'LongLong': 'int64', +- 'Int': 'int32', +- 'Double': 'float64' ++sizeof_long = array.array('l').itemsize ++deprecated_name_map = { ++ # deprecated: replacement ++ 'Float': 'Float32', ++ 'Double': 'Float64', ++ 'Char': 'Int8', ++ 'Int': 'Int32', ++ 'Long': 'Int32' if sizeof_long == 4 else 'Int64', ++ 'LongLong': 'Int64', ++ 'Byte': 'UInt8', ++ # previously misspelled variant ++ 'Uint64': 'UInt64', + } + +-sizeof_long = array.array('l').itemsize +-if sizeof_long == 4: +- vector_name_map["Long"] = 'int32' +-elif sizeof_long == 8: +- vector_name_map["Long"] = 'int64' ++for depr_prefix, base_prefix in deprecated_name_map.items(): ++ _make_deprecated_swig_class(depr_prefix + "Vector", base_prefix + "Vector") ++ ++ # same for the three legacy *VectorVector classes ++ if depr_prefix in ['Float', 'Long', 'Byte']: ++ _make_deprecated_swig_class(depr_prefix + "VectorVector", ++ base_prefix + "VectorVector") + ++# mapping from vector names in swigfaiss.swig and the numpy dtype names ++# TODO: once deprecated classes are removed, remove the dict and just use .lower() below ++vector_name_map = { ++ 'Float32': 'float32', ++ 'Float64': 'float64', ++ 'Int8': 'int8', ++ 'Int16': 'int16', ++ 'Int32': 'int32', ++ 'Int64': 'int64', ++ 'UInt8': 'uint8', ++ 'UInt16': 'uint16', ++ 'UInt32': 'uint32', ++ 'UInt64': 'uint64', ++ **{k: v.lower() for k, v in deprecated_name_map.items()} ++} + + + def vector_to_array(v): +diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig +index 3113a136..5f9302d3 100644 +--- a/faiss/python/swigfaiss.swig ++++ b/faiss/python/swigfaiss.swig +@@ -200,23 +200,21 @@ namespace std { + %include + + // primitive array types +-%template(FloatVector) std::vector; +-%template(DoubleVector) std::vector; +-%template(ByteVector) std::vector; +-%template(CharVector) std::vector; +-// NOTE(hoss): Using unsigned long instead of uint64_t because OSX defines +-// uint64_t as unsigned long long, which SWIG is not aware of. +-%template(Uint64Vector) std::vector; +-%template(LongVector) std::vector; +-%template(LongLongVector) std::vector; +-%template(IntVector) std::vector; +-%template(UInt32Vector) std::vector; ++%template(Float32Vector) std::vector; ++%template(Float64Vector) std::vector; ++%template(Int8Vector) std::vector; + %template(Int16Vector) std::vector; ++%template(Int32Vector) std::vector; ++%template(Int64Vector) std::vector; ++%template(UInt8Vector) std::vector; + %template(UInt16Vector) std::vector; ++%template(UInt32Vector) std::vector; ++%template(UInt64Vector) std::vector; + +-%template(FloatVectorVector) std::vector >; +-%template(ByteVectorVector) std::vector >; +-%template(LongVectorVector) std::vector >; ++%template(Float32VectorVector) std::vector >; ++%template(UInt8VectorVector) std::vector >; ++%template(Int32VectorVector) std::vector >; ++%template(Int64VectorVector) std::vector >; + %template(VectorTransformVector) std::vector; + %template(OperatingPointVector) std::vector; + %template(InvertedListsPtrVector) std::vector; +@@ -300,9 +298,9 @@ void gpu_sync_all_devices() + + %} + +-%template() std::pair; +-%template() std::map >; +-%template() std::map > >; ++%template() std::pair; ++%template() std::map >; ++%template() std::map > >; + + // causes weird wrapper bug + %ignore *::allocMemoryHandle; +@@ -317,12 +315,12 @@ typedef CUstream_st* cudaStream_t; + %inline %{ + + // interop between pytorch exposed cudaStream_t and faiss +-cudaStream_t cast_integer_to_cudastream_t(long long x) { ++cudaStream_t cast_integer_to_cudastream_t(int64_t x) { + return (cudaStream_t) x; + } + +-long long cast_cudastream_t_to_integer(cudaStream_t x) { +- return (long) x; ++int64_t cast_cudastream_t_to_integer(cudaStream_t x) { ++ return (int64_t) x; + } + + %} +@@ -898,23 +896,23 @@ void *memcpy(void *dest, const void *src, size_t n); + *******************************************************************/ + + %inline %{ +-uint8_t * cast_integer_to_uint8_ptr (long long x) { ++uint8_t * cast_integer_to_uint8_ptr (int64_t x) { + return (uint8_t*)x; + } + +-float * cast_integer_to_float_ptr (long long x) { ++float * cast_integer_to_float_ptr (int64_t x) { + return (float*)x; + } + +-faiss::Index::idx_t* cast_integer_to_idx_t_ptr (long long x) { ++faiss::Index::idx_t* cast_integer_to_idx_t_ptr (int64_t x) { + return (faiss::Index::idx_t*)x; + } + +-int * cast_integer_to_int_ptr (long long x) { ++int * cast_integer_to_int_ptr (int64_t x) { + return (int*)x; + } + +-void * cast_integer_to_void_ptr (long long x) { ++void * cast_integer_to_void_ptr (int64_t x) { + return (void*)x; + } + +@@ -941,7 +939,7 @@ struct MapLong2Long { + } + } + +- long search(int64_t key) { ++ int64_t search(int64_t key) { + if (map.count(key) == 0) { + return -1; + } else { +diff --git a/tests/test_build_blocks.py b/tests/test_build_blocks.py +index 713ed8fb..76135d9c 100644 +--- a/tests/test_build_blocks.py ++++ b/tests/test_build_blocks.py +@@ -614,13 +614,7 @@ class TestSWIGWrap(unittest.TestCase): + + def test_int64(self): + # see https://github.com/facebookresearch/faiss/issues/1529 +- sizeof_long = array.array("l").itemsize +- if sizeof_long == 4: +- v = faiss.LongLongVector() +- elif sizeof_long == 8: +- v = faiss.LongVector() +- else: +- raise AssertionError("weird long size") ++ v = faiss.Int64Vector() + + for i in range(10): + v.push_back(i) +diff --git a/tests/test_oom_exception.py b/tests/test_oom_exception.py +index 341004d9..e5300884 100644 +--- a/tests/test_oom_exception.py ++++ b/tests/test_oom_exception.py +@@ -19,7 +19,7 @@ class TestOOMException(unittest.TestCase): + # make sure that allocing more than 10G will fail + resource.setrlimit(resource.RLIMIT_AS, (10 * 1024 * 1024, hard_as)) + try: +- x = faiss.IntVector() ++ x = faiss.Int32Vector() + try: + x.resize(10**11) # 400 G of RAM + except MemoryError: +-- +2.29.2.windows.3 + diff --git a/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch b/recipe/patches/0015-adapt-header-target-directory-to-outputname.patch similarity index 95% rename from recipe/patches/0012-adapt-header-target-directory-to-outputname.patch rename to recipe/patches/0015-adapt-header-target-directory-to-outputname.patch index 295e6d5e..5aa57066 100644 --- a/recipe/patches/0012-adapt-header-target-directory-to-outputname.patch +++ b/recipe/patches/0015-adapt-header-target-directory-to-outputname.patch @@ -1,7 +1,7 @@ -From 84072b99d78be9d94bf902e66fff396c1b94b21b Mon Sep 17 00:00:00 2001 +From 7be924513f7590ae65131a820a04287d457e2c3d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sun, 14 Feb 2021 15:52:27 +0100 -Subject: [PATCH 12/18] adapt header target directory to outputname +Subject: [PATCH 15/16] adapt header target directory to outputname --- faiss/CMakeLists.txt | 11 ++++++++--- diff --git a/recipe/patches/0016-skip-test_stress-for-GPU-on-windows.patch b/recipe/patches/0016-skip-test_stress-for-GPU-on-windows.patch index 214e7b72..afae7781 100644 --- a/recipe/patches/0016-skip-test_stress-for-GPU-on-windows.patch +++ b/recipe/patches/0016-skip-test_stress-for-GPU-on-windows.patch @@ -1,7 +1,7 @@ -From 8e93a51907db47ba8be7feb3b18db876f199c6d1 Mon Sep 17 00:00:00 2001 +From 1bb5ff3db58eb37a3ea61bc99d6a9e7786e8f27e Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 19 Feb 2021 14:14:31 +0100 -Subject: [PATCH 16/18] skip test_stress for GPU on windows +Subject: [PATCH 16/16] skip test_stress for GPU on windows --- faiss/gpu/test/test_gpu_index.py | 2 ++ diff --git a/recipe/patches/0017-handle-long-vs.-windows-in-swigfaiss.swig.patch b/recipe/patches/0017-handle-long-vs.-windows-in-swigfaiss.swig.patch deleted file mode 100644 index 2d1181d9..00000000 --- a/recipe/patches/0017-handle-long-vs.-windows-in-swigfaiss.swig.patch +++ /dev/null @@ -1,127 +0,0 @@ -From 11e223ada29dfa6002c1dda335c73957be07a463 Mon Sep 17 00:00:00 2001 -From: "H. Vetinari" -Date: Wed, 3 Mar 2021 23:59:39 +0100 -Subject: [PATCH 17/18] handle long vs. windows in swigfaiss.swig - ---- - faiss/python/swigfaiss.swig | 40 ++++++++++++++++++------------------- - tests/test_build_blocks.py | 4 +--- - 2 files changed, 21 insertions(+), 23 deletions(-) - -diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig -index 3113a136..3086ffab 100644 ---- a/faiss/python/swigfaiss.swig -+++ b/faiss/python/swigfaiss.swig -@@ -204,19 +204,19 @@ namespace std { - %template(DoubleVector) std::vector; - %template(ByteVector) std::vector; - %template(CharVector) std::vector; --// NOTE(hoss): Using unsigned long instead of uint64_t because OSX defines --// uint64_t as unsigned long long, which SWIG is not aware of. --%template(Uint64Vector) std::vector; --%template(LongVector) std::vector; --%template(LongLongVector) std::vector; --%template(IntVector) std::vector; --%template(UInt32Vector) std::vector; - %template(Int16Vector) std::vector; -+// consider renaming to Int32Vector / Int64Vector -+%template(IntVector) std::vector; -+%template(LongVector) std::vector; -+// TODO: deprecate/remove LongLongVector -+%template(LongLongVector) std::vector; - %template(UInt16Vector) std::vector; -+%template(UInt32Vector) std::vector; -+%template(Uint64Vector) std::vector; - - %template(FloatVectorVector) std::vector >; - %template(ByteVectorVector) std::vector >; --%template(LongVectorVector) std::vector >; -+%template(LongVectorVector) std::vector >; - %template(VectorTransformVector) std::vector; - %template(OperatingPointVector) std::vector; - %template(InvertedListsPtrVector) std::vector; -@@ -300,9 +300,9 @@ void gpu_sync_all_devices() - - %} - --%template() std::pair; --%template() std::map >; --%template() std::map > >; -+%template() std::pair; -+%template() std::map >; -+%template() std::map > >; - - // causes weird wrapper bug - %ignore *::allocMemoryHandle; -@@ -317,12 +317,12 @@ typedef CUstream_st* cudaStream_t; - %inline %{ - - // interop between pytorch exposed cudaStream_t and faiss --cudaStream_t cast_integer_to_cudastream_t(long long x) { -+cudaStream_t cast_integer_to_cudastream_t(int64_t x) { - return (cudaStream_t) x; - } - --long long cast_cudastream_t_to_integer(cudaStream_t x) { -- return (long) x; -+int64_t cast_cudastream_t_to_integer(cudaStream_t x) { -+ return (int64_t) x; - } - - %} -@@ -898,23 +898,23 @@ void *memcpy(void *dest, const void *src, size_t n); - *******************************************************************/ - - %inline %{ --uint8_t * cast_integer_to_uint8_ptr (long long x) { -+uint8_t * cast_integer_to_uint8_ptr (int64_t x) { - return (uint8_t*)x; - } - --float * cast_integer_to_float_ptr (long long x) { -+float * cast_integer_to_float_ptr (int64_t x) { - return (float*)x; - } - --faiss::Index::idx_t* cast_integer_to_idx_t_ptr (long long x) { -+faiss::Index::idx_t* cast_integer_to_idx_t_ptr (int64_t x) { - return (faiss::Index::idx_t*)x; - } - --int * cast_integer_to_int_ptr (long long x) { -+int * cast_integer_to_int_ptr (int64_t x) { - return (int*)x; - } - --void * cast_integer_to_void_ptr (long long x) { -+void * cast_integer_to_void_ptr (int64_t x) { - return (void*)x; - } - -@@ -941,7 +941,7 @@ struct MapLong2Long { - } - } - -- long search(int64_t key) { -+ int64_t search(int64_t key) { - if (map.count(key) == 0) { - return -1; - } else { -diff --git a/tests/test_build_blocks.py b/tests/test_build_blocks.py -index 713ed8fb..dd6b5961 100644 ---- a/tests/test_build_blocks.py -+++ b/tests/test_build_blocks.py -@@ -615,9 +615,7 @@ class TestSWIGWrap(unittest.TestCase): - def test_int64(self): - # see https://github.com/facebookresearch/faiss/issues/1529 - sizeof_long = array.array("l").itemsize -- if sizeof_long == 4: -- v = faiss.LongLongVector() -- elif sizeof_long == 8: -+ if sizeof_long in (4, 8): - v = faiss.LongVector() - else: - raise AssertionError("weird long size") --- -2.29.2.windows.3 - diff --git a/recipe/patches/0018-deprecate-non-portable-Vector-names-and-replace-with.patch b/recipe/patches/0018-deprecate-non-portable-Vector-names-and-replace-with.patch deleted file mode 100644 index 104d9b58..00000000 --- a/recipe/patches/0018-deprecate-non-portable-Vector-names-and-replace-with.patch +++ /dev/null @@ -1,179 +0,0 @@ -From 348534a9950ccfc5da53fae04d95aceb344fc378 Mon Sep 17 00:00:00 2001 -From: "H. Vetinari" -Date: Fri, 5 Mar 2021 09:54:40 +0100 -Subject: [PATCH 18/18] deprecate non-portable Vector names and replace with - clearer nomenclature - ---- - faiss/python/__init__.py | 63 +++++++++++++++++++++++++++---------- - faiss/python/swigfaiss.swig | 21 ++++++------- - tests/test_build_blocks.py | 5 +-- - tests/test_oom_exception.py | 2 +- - 4 files changed, 58 insertions(+), 33 deletions(-) - -diff --git a/faiss/python/__init__.py b/faiss/python/__init__.py -index 315871aa..dbc1a8f2 100644 ---- a/faiss/python/__init__.py -+++ b/faiss/python/__init__.py -@@ -5,13 +5,14 @@ - - #@nolint - --# not linting this file because it imports * form swigfaiss, which -+# not linting this file because it imports * from swigfaiss, which - # causes a ton of useless warnings. - - import numpy as np - import sys - import inspect - import array -+import warnings - - # We import * so that the symbol foo can be accessed as faiss.foo. - from .loader import * -@@ -523,7 +524,7 @@ def index_cpu_to_gpu_multiple_py(resources, index, co=None, gpus=None): - if gpus is None: - gpus = range(len(resources)) - vres = GpuResourcesVector() -- vdev = IntVector() -+ vdev = Int32Vector() - for i, res in zip(gpus, resources): - vdev.push_back(i) - vres.push_back(res) -@@ -635,23 +636,53 @@ def knn_gpu(res, xb, xq, k, D=None, I=None, metric=METRIC_L2): - # numpy array / std::vector conversions - ########################################### - --# mapping from vector names in swigfaiss.swig and the numpy dtype names --vector_name_map = { -- 'Float': 'float32', -- 'Byte': 'uint8', -- 'Char': 'int8', -- 'Uint64': 'uint64', -- 'LongLong': 'int64', -- 'Int': 'int32', -- 'Double': 'float64' -+sizeof_long = array.array('l').itemsize -+deprecated_name_map = { -+ 'Float': 'Float32', -+ 'Double': 'Float64', -+ 'Char': 'Int8', -+ 'Int': 'Int32', -+ 'Long': 'Int32' if sizeof_long == 4 else 'Int64', -+ 'LongLong': 'Int64', -+ 'Byte': 'UInt8' - } - --sizeof_long = array.array('l').itemsize --if sizeof_long == 4: -- vector_name_map["Long"] = 'int32' --elif sizeof_long == 8: -- vector_name_map["Long"] = 'int64' -+def make_deprecated_class(deprecated_name, base_name): -+ """ -+ Dynamically construct deprecated classes and add warning to their __new__-method. -+ -+ We do this here because the classes are defined in the swig interface, -+ making it cumbersome to add the following functionality there directly. -+ """ -+ base_class = globals()[base_name] -+ def new_meth(cls, *args, **kwargs): -+ msg = f"The class faiss.{deprecated_name} is deprecated in favour of faiss.{base_name}!" -+ warnings.warn(msg, DeprecationWarning, stacklevel=2) -+ instance = super(base_class, cls).__new__(cls, *args, **kwargs) -+ return instance - -+ # three-argument version of type uses (name, tuple-of-bases, dict-of-attributes) -+ return type(deprecated_name, (base_class,), {"__new__": new_meth}) -+ -+for depr_prefix, base_prefix in deprecated_name_map.items(): -+ klazz = make_deprecated_class(depr_prefix + "Vector", base_prefix + "Vector") -+ globals()[depr_prefix + "Vector"] = klazz -+ -+# mapping from vector names in swigfaiss.swig and the numpy dtype names -+# TODO: once deprecated classes are removed, remove the dict and just use .lower() below -+vector_name_map = { -+ 'Float32': 'float32', -+ 'Float64': 'float64', -+ 'Int8': 'int8', -+ 'Int16': 'int16', -+ 'Int32': 'int32', -+ 'Int64': 'int64', -+ 'UInt8': 'uint8', -+ 'UInt16': 'uint16', -+ 'UInt32': 'uint32', -+ 'UInt64': 'uint64', -+ **{k: v.lower() for k, v in deprecated_name_map.items()} -+} - - - def vector_to_array(v): -diff --git a/faiss/python/swigfaiss.swig b/faiss/python/swigfaiss.swig -index 3086ffab..1e7bdf3b 100644 ---- a/faiss/python/swigfaiss.swig -+++ b/faiss/python/swigfaiss.swig -@@ -200,23 +200,20 @@ namespace std { - %include - - // primitive array types --%template(FloatVector) std::vector; --%template(DoubleVector) std::vector; --%template(ByteVector) std::vector; --%template(CharVector) std::vector; -+%template(Float32Vector) std::vector; -+%template(Float64Vector) std::vector; -+%template(Int8Vector) std::vector; - %template(Int16Vector) std::vector; --// consider renaming to Int32Vector / Int64Vector --%template(IntVector) std::vector; --%template(LongVector) std::vector; --// TODO: deprecate/remove LongLongVector --%template(LongLongVector) std::vector; -+%template(Int32Vector) std::vector; -+%template(Int64Vector) std::vector; -+%template(UInt8Vector) std::vector; - %template(UInt16Vector) std::vector; - %template(UInt32Vector) std::vector; - %template(Uint64Vector) std::vector; - --%template(FloatVectorVector) std::vector >; --%template(ByteVectorVector) std::vector >; --%template(LongVectorVector) std::vector >; -+%template(Float32VectorVector) std::vector >; -+%template(UInt8VectorVector) std::vector >; -+%template(Int64VectorVector) std::vector >; - %template(VectorTransformVector) std::vector; - %template(OperatingPointVector) std::vector; - %template(InvertedListsPtrVector) std::vector; -diff --git a/tests/test_build_blocks.py b/tests/test_build_blocks.py -index dd6b5961..5534f39a 100644 ---- a/tests/test_build_blocks.py -+++ b/tests/test_build_blocks.py -@@ -615,10 +615,7 @@ class TestSWIGWrap(unittest.TestCase): - def test_int64(self): - # see https://github.com/facebookresearch/faiss/issues/1529 - sizeof_long = array.array("l").itemsize -- if sizeof_long in (4, 8): -- v = faiss.LongVector() -- else: -- raise AssertionError("weird long size") -+ v = faiss.Int64Vector() - - for i in range(10): - v.push_back(i) -diff --git a/tests/test_oom_exception.py b/tests/test_oom_exception.py -index 341004d9..e5300884 100644 ---- a/tests/test_oom_exception.py -+++ b/tests/test_oom_exception.py -@@ -19,7 +19,7 @@ class TestOOMException(unittest.TestCase): - # make sure that allocing more than 10G will fail - resource.setrlimit(resource.RLIMIT_AS, (10 * 1024 * 1024, hard_as)) - try: -- x = faiss.IntVector() -+ x = faiss.Int32Vector() - try: - x.resize(10**11) # 400 G of RAM - except MemoryError: --- -2.29.2.windows.3 - From ec4bbeeab124d5fcfaba3d6e7c383b23a53995ff Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 6 Apr 2021 18:39:53 +0200 Subject: [PATCH 12/12] MNT: Re-rendered with conda-build 3.21.4, conda-smithy 3.9.0, and conda-forge-pinning 2021.04.06.13.31.20 --- .azure-pipelines/azure-pipelines-win.yml | 12 ++++++ .../linux_64_cuda_compiler_version10.2.yaml | 6 +-- .../linux_64_cuda_compiler_version11.0.yaml | 6 +-- .../linux_64_cuda_compiler_version11.1.yaml | 6 +-- .../linux_64_cuda_compiler_version11.2.yaml | 6 +-- .../linux_64_cuda_compiler_versionNone.yaml | 6 +-- .ci_support/osx_64_.yaml | 6 +-- .../win_64_cuda_compiler_version10.2.yaml | 33 +++++++++++++++++ .../win_64_cuda_compiler_version11.0.yaml | 33 +++++++++++++++++ .../win_64_cuda_compiler_version11.1.yaml | 33 +++++++++++++++++ .../win_64_cuda_compiler_version11.2.yaml | 33 +++++++++++++++++ .../win_64_cuda_compiler_versionNone.yaml | 8 ++-- .scripts/build_steps.sh | 22 +++++++++-- .scripts/logging_utils.sh | 30 +++++++++++++++ .scripts/run_docker_build.sh | 6 +++ .scripts/run_osx_build.sh | 34 ++++++++--------- LICENSE.txt | 2 +- README.md | 37 +++++++++++++++++-- 18 files changed, 272 insertions(+), 47 deletions(-) create mode 100644 .ci_support/win_64_cuda_compiler_version10.2.yaml create mode 100644 .ci_support/win_64_cuda_compiler_version11.0.yaml create mode 100644 .ci_support/win_64_cuda_compiler_version11.1.yaml create mode 100644 .ci_support/win_64_cuda_compiler_version11.2.yaml create mode 100644 .scripts/logging_utils.sh diff --git a/.azure-pipelines/azure-pipelines-win.yml b/.azure-pipelines/azure-pipelines-win.yml index 31bd165b..b28c5326 100755 --- a/.azure-pipelines/azure-pipelines-win.yml +++ b/.azure-pipelines/azure-pipelines-win.yml @@ -8,6 +8,18 @@ jobs: vmImage: windows-2019 strategy: matrix: + win_64_cuda_compiler_version10.2: + CONFIG: win_64_cuda_compiler_version10.2 + UPLOAD_PACKAGES: 'True' + win_64_cuda_compiler_version11.0: + CONFIG: win_64_cuda_compiler_version11.0 + UPLOAD_PACKAGES: 'True' + win_64_cuda_compiler_version11.1: + CONFIG: win_64_cuda_compiler_version11.1 + UPLOAD_PACKAGES: 'True' + win_64_cuda_compiler_version11.2: + CONFIG: win_64_cuda_compiler_version11.2 + UPLOAD_PACKAGES: 'True' win_64_cuda_compiler_versionNone: CONFIG: win_64_cuda_compiler_versionNone UPLOAD_PACKAGES: 'True' diff --git a/.ci_support/linux_64_cuda_compiler_version10.2.yaml b/.ci_support/linux_64_cuda_compiler_version10.2.yaml index 8325e197..ba91a134 100644 --- a/.ci_support/linux_64_cuda_compiler_version10.2.yaml +++ b/.ci_support/linux_64_cuda_compiler_version10.2.yaml @@ -19,9 +19,9 @@ libblas: liblapack: - 3.8 *netlib numpy: -- '1.16' -- '1.16' -- '1.16' +- '1.17' +- '1.17' +- '1.17' - '1.19' pin_run_as_build: python: diff --git a/.ci_support/linux_64_cuda_compiler_version11.0.yaml b/.ci_support/linux_64_cuda_compiler_version11.0.yaml index f738e8f7..0b169c48 100644 --- a/.ci_support/linux_64_cuda_compiler_version11.0.yaml +++ b/.ci_support/linux_64_cuda_compiler_version11.0.yaml @@ -19,9 +19,9 @@ libblas: liblapack: - 3.8 *netlib numpy: -- '1.16' -- '1.16' -- '1.16' +- '1.17' +- '1.17' +- '1.17' - '1.19' pin_run_as_build: python: diff --git a/.ci_support/linux_64_cuda_compiler_version11.1.yaml b/.ci_support/linux_64_cuda_compiler_version11.1.yaml index 3de25556..63076408 100644 --- a/.ci_support/linux_64_cuda_compiler_version11.1.yaml +++ b/.ci_support/linux_64_cuda_compiler_version11.1.yaml @@ -19,9 +19,9 @@ libblas: liblapack: - 3.8 *netlib numpy: -- '1.16' -- '1.16' -- '1.16' +- '1.17' +- '1.17' +- '1.17' - '1.19' pin_run_as_build: python: diff --git a/.ci_support/linux_64_cuda_compiler_version11.2.yaml b/.ci_support/linux_64_cuda_compiler_version11.2.yaml index 3e92de38..3c4804aa 100644 --- a/.ci_support/linux_64_cuda_compiler_version11.2.yaml +++ b/.ci_support/linux_64_cuda_compiler_version11.2.yaml @@ -19,9 +19,9 @@ libblas: liblapack: - 3.8 *netlib numpy: -- '1.16' -- '1.16' -- '1.16' +- '1.17' +- '1.17' +- '1.17' - '1.19' pin_run_as_build: python: diff --git a/.ci_support/linux_64_cuda_compiler_versionNone.yaml b/.ci_support/linux_64_cuda_compiler_versionNone.yaml index 48130be9..f79c6247 100644 --- a/.ci_support/linux_64_cuda_compiler_versionNone.yaml +++ b/.ci_support/linux_64_cuda_compiler_versionNone.yaml @@ -19,9 +19,9 @@ libblas: liblapack: - 3.8 *netlib numpy: -- '1.16' -- '1.16' -- '1.16' +- '1.17' +- '1.17' +- '1.17' - '1.19' pin_run_as_build: python: diff --git a/.ci_support/osx_64_.yaml b/.ci_support/osx_64_.yaml index 6768d9d9..77fd5c85 100644 --- a/.ci_support/osx_64_.yaml +++ b/.ci_support/osx_64_.yaml @@ -17,9 +17,9 @@ liblapack: macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.16' -- '1.16' -- '1.16' +- '1.17' +- '1.17' +- '1.17' - '1.19' pin_run_as_build: python: diff --git a/.ci_support/win_64_cuda_compiler_version10.2.yaml b/.ci_support/win_64_cuda_compiler_version10.2.yaml new file mode 100644 index 00000000..e983aa1b --- /dev/null +++ b/.ci_support/win_64_cuda_compiler_version10.2.yaml @@ -0,0 +1,33 @@ +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.2' +cxx_compiler: +- vs2019 +libblas: +- 3.8 *netlib +liblapack: +- 3.8 *netlib +numpy: +- '1.17' +- '1.17' +- '1.17' +- '1.19' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython +- 3.7.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +target_platform: +- win-64 +zip_keys: +- - numpy + - python diff --git a/.ci_support/win_64_cuda_compiler_version11.0.yaml b/.ci_support/win_64_cuda_compiler_version11.0.yaml new file mode 100644 index 00000000..3c7f5c24 --- /dev/null +++ b/.ci_support/win_64_cuda_compiler_version11.0.yaml @@ -0,0 +1,33 @@ +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '11.0' +cxx_compiler: +- vs2019 +libblas: +- 3.8 *netlib +liblapack: +- 3.8 *netlib +numpy: +- '1.17' +- '1.17' +- '1.17' +- '1.19' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython +- 3.7.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +target_platform: +- win-64 +zip_keys: +- - numpy + - python diff --git a/.ci_support/win_64_cuda_compiler_version11.1.yaml b/.ci_support/win_64_cuda_compiler_version11.1.yaml new file mode 100644 index 00000000..a5750d71 --- /dev/null +++ b/.ci_support/win_64_cuda_compiler_version11.1.yaml @@ -0,0 +1,33 @@ +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '11.1' +cxx_compiler: +- vs2019 +libblas: +- 3.8 *netlib +liblapack: +- 3.8 *netlib +numpy: +- '1.17' +- '1.17' +- '1.17' +- '1.19' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython +- 3.7.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +target_platform: +- win-64 +zip_keys: +- - numpy + - python diff --git a/.ci_support/win_64_cuda_compiler_version11.2.yaml b/.ci_support/win_64_cuda_compiler_version11.2.yaml new file mode 100644 index 00000000..7499913e --- /dev/null +++ b/.ci_support/win_64_cuda_compiler_version11.2.yaml @@ -0,0 +1,33 @@ +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '11.2' +cxx_compiler: +- vs2019 +libblas: +- 3.8 *netlib +liblapack: +- 3.8 *netlib +numpy: +- '1.17' +- '1.17' +- '1.17' +- '1.19' +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.6.* *_cpython +- 3.7.* *_cpython +- 3.8.* *_cpython +- 3.9.* *_cpython +target_platform: +- win-64 +zip_keys: +- - numpy + - python diff --git a/.ci_support/win_64_cuda_compiler_versionNone.yaml b/.ci_support/win_64_cuda_compiler_versionNone.yaml index f69d63eb..2e8de7e7 100644 --- a/.ci_support/win_64_cuda_compiler_versionNone.yaml +++ b/.ci_support/win_64_cuda_compiler_versionNone.yaml @@ -2,6 +2,8 @@ channel_sources: - conda-forge,defaults channel_targets: - conda-forge main +cuda_compiler: +- nvcc cuda_compiler_version: - None cxx_compiler: @@ -11,9 +13,9 @@ libblas: liblapack: - 3.8 *netlib numpy: -- '1.16' -- '1.16' -- '1.16' +- '1.17' +- '1.17' +- '1.17' - '1.19' pin_run_as_build: python: diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index aa9727b9..522eabcf 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -6,8 +6,14 @@ # benefit from the improvement. set -xeuo pipefail -export PYTHONUNBUFFERED=1 export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" +source ${FEEDSTOCK_ROOT}/.scripts/logging_utils.sh + + +endgroup "Start Docker" + +startgroup "Configuring conda" +export PYTHONUNBUFFERED=1 export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" @@ -18,8 +24,9 @@ conda-build: root-dir: ${FEEDSTOCK_ROOT}/build_artifacts CONDARC +BUILD_CMD=build -conda install --yes --quiet "conda-forge-ci-setup=3" conda-build pip -c conda-forge +conda install --yes --quiet "conda-forge-ci-setup=3" conda-build pip ${GET_BOA:-} -c conda-forge # set up the condarc setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" @@ -29,24 +36,33 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" +endgroup "Configuring conda" if [[ "${BUILD_WITH_CONDA_DEBUG:-0}" == 1 ]]; then + startgroup "Running conda debug" if [[ "x${BUILD_OUTPUT_ID:-}" != "x" ]]; then EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --output-id ${BUILD_OUTPUT_ID}" fi conda debug "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ ${EXTRA_CB_OPTIONS:-} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" + endgroup "Running conda debug" # Drop into an interactive shell /bin/bash else - conda build "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ + startgroup "Running conda $BUILD_CMD" + conda $BUILD_CMD "${RECIPE_ROOT}" -m "${CI_SUPPORT}/${CONFIG}.yaml" \ --suppress-variables ${EXTRA_CB_OPTIONS:-} \ --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" + endgroup "Running conda build" + startgroup "Validating outputs" validate_recipe_outputs "${FEEDSTOCK_NAME}" + endgroup "Validating outputs" if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then + startgroup "Uploading packages" upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" + endgroup "Uploading packages" fi fi diff --git a/.scripts/logging_utils.sh b/.scripts/logging_utils.sh new file mode 100644 index 00000000..a53ef3f2 --- /dev/null +++ b/.scripts/logging_utils.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Provide a unified interface for the different logging +# utilities CI providers offer. If unavailable, provide +# a compatible fallback (e.g. bare `echo xxxxxx`). + +function startgroup { + # Start a foldable group of log lines + # Pass a single argument, quoted + case ${CI:-} in + azure ) + echo "##[group]$1";; + travis ) + echo "$1" + echo -en 'travis_fold:start:'"${1// /}"'\\r';; + * ) + echo "$1";; + esac +} + +function endgroup { + # End a foldable group of log lines + # Pass a single argument, quoted + case ${CI:-} in + azure ) + echo "##[endgroup]";; + travis ) + echo -en 'travis_fold:end:'"${1// /}"'\\r';; + esac +} diff --git a/.scripts/run_docker_build.sh b/.scripts/run_docker_build.sh index 36dacd6e..16d6c15a 100755 --- a/.scripts/run_docker_build.sh +++ b/.scripts/run_docker_build.sh @@ -5,6 +5,10 @@ # changes to this script, consider a proposal to conda-smithy so that other feedstocks can also # benefit from the improvement. +source .scripts/logging_utils.sh + +startgroup "Configure Docker" + set -xeo pipefail THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )" @@ -65,7 +69,9 @@ DOCKER_RUN_ARGS="${CONDA_FORGE_DOCKER_RUN_ARGS}" if [ -z "${CI}" ]; then DOCKER_RUN_ARGS="-it ${DOCKER_RUN_ARGS}" fi +endgroup "Configure Docker" +startgroup "Start Docker" export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z,delegated \ diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index c299b792..614ca844 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -1,29 +1,24 @@ #!/usr/bin/env bash +source .scripts/logging_utils.sh + set -x -echo -e "\n\nInstalling a fresh version of Miniforge." -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:start:install_miniforge\\r' -fi +startgroup "Installing a fresh version of Miniforge" MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download" MINIFORGE_FILE="Miniforge3-MacOSX-x86_64.sh" curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}" bash $MINIFORGE_FILE -b -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:end:install_miniforge\\r' -fi +endgroup "Installing a fresh version of Miniforge" -echo -e "\n\nConfiguring conda." -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:start:configure_conda\\r' -fi +startgroup "Configuring conda" +BUILD_CMD=build source ${HOME}/miniforge3/etc/profile.d/conda.sh conda activate base echo -e "\n\nInstalling conda-forge-ci-setup=3 and conda-build." -conda install -n base --quiet --yes "conda-forge-ci-setup=3" conda-build pip +conda install -n base --quiet --yes "conda-forge-ci-setup=3" conda-build pip ${GET_BOA:-} @@ -39,19 +34,22 @@ echo -e "\n\nRunning the build setup script." source run_conda_forge_build_setup -if [[ ${CI} == "travis" ]]; then - echo -en 'travis_fold:end:configure_conda\\r' -fi +endgroup "Configuring conda" set -e -echo -e "\n\nMaking the build clobber file and running the build." +startgroup "Running conda $BUILD_CMD" +echo -e "\n\nMaking the build clobber file" make_build_number ./ ./recipe ./.ci_support/${CONFIG}.yaml -conda build ./recipe -m ./.ci_support/${CONFIG}.yaml --suppress-variables --clobber-file ./.ci_support/clobber_${CONFIG}.yaml ${EXTRA_CB_OPTIONS:-} +conda $BUILD_CMD ./recipe -m ./.ci_support/${CONFIG}.yaml --suppress-variables --clobber-file ./.ci_support/clobber_${CONFIG}.yaml ${EXTRA_CB_OPTIONS:-} +endgroup "Running conda build" +startgroup "Validating outputs" validate_recipe_outputs "${FEEDSTOCK_NAME}" +endgroup "Validating outputs" if [[ "${UPLOAD_PACKAGES}" != "False" ]]; then - echo -e "\n\nUploading the packages." + startgroup "Uploading packages" upload_package --validate --feedstock-name="${FEEDSTOCK_NAME}" ./ ./recipe ./.ci_support/${CONFIG}.yaml + endgroup "Uploading packages" fi \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt index 5f302793..ed3f451e 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,5 +1,5 @@ BSD 3-clause license -Copyright (c) 2015-2020, conda-forge contributors +Copyright (c) 2015-2021, conda-forge contributors All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index 546a045c..a7c79e96 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,34 @@ Current build status variant + + win_64_cuda_compiler_version10.2 + + + variant + + + + win_64_cuda_compiler_version11.0 + + + variant + + + + win_64_cuda_compiler_version11.1 + + + variant + + + + win_64_cuda_compiler_version11.2 + + + variant + + win_64_cuda_compiler_versionNone @@ -110,6 +138,7 @@ Current release info | [![Conda Recipe](https://img.shields.io/badge/recipe-faiss--gpu-green.svg)](https://anaconda.org/conda-forge/faiss-gpu) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/faiss-gpu.svg)](https://anaconda.org/conda-forge/faiss-gpu) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/faiss-gpu.svg)](https://anaconda.org/conda-forge/faiss-gpu) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/faiss-gpu.svg)](https://anaconda.org/conda-forge/faiss-gpu) | | [![Conda Recipe](https://img.shields.io/badge/recipe-faiss--proc-green.svg)](https://anaconda.org/conda-forge/faiss-proc) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/faiss-proc.svg)](https://anaconda.org/conda-forge/faiss-proc) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/faiss-proc.svg)](https://anaconda.org/conda-forge/faiss-proc) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/faiss-proc.svg)](https://anaconda.org/conda-forge/faiss-proc) | | [![Conda Recipe](https://img.shields.io/badge/recipe-libfaiss-green.svg)](https://anaconda.org/conda-forge/libfaiss) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/libfaiss.svg)](https://anaconda.org/conda-forge/libfaiss) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/libfaiss.svg)](https://anaconda.org/conda-forge/libfaiss) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/libfaiss.svg)](https://anaconda.org/conda-forge/libfaiss) | +| [![Conda Recipe](https://img.shields.io/badge/recipe-libfaiss--avx2-green.svg)](https://anaconda.org/conda-forge/libfaiss-avx2) | [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/libfaiss-avx2.svg)](https://anaconda.org/conda-forge/libfaiss-avx2) | [![Conda Version](https://img.shields.io/conda/vn/conda-forge/libfaiss-avx2.svg)](https://anaconda.org/conda-forge/libfaiss-avx2) | [![Conda Platforms](https://img.shields.io/conda/pn/conda-forge/libfaiss-avx2.svg)](https://anaconda.org/conda-forge/libfaiss-avx2) | Installing faiss-split ====================== @@ -120,10 +149,10 @@ Installing `faiss-split` from the `conda-forge` channel can be achieved by addin conda config --add channels conda-forge ``` -Once the `conda-forge` channel has been enabled, `faiss, faiss-cpu, faiss-gpu, faiss-proc, libfaiss` can be installed with: +Once the `conda-forge` channel has been enabled, `faiss, faiss-cpu, faiss-gpu, faiss-proc, libfaiss, libfaiss-avx2` can be installed with: ``` -conda install faiss faiss-cpu faiss-gpu faiss-proc libfaiss +conda install faiss faiss-cpu faiss-gpu faiss-proc libfaiss libfaiss-avx2 ``` It is possible to list all of the versions of `faiss` available on your platform with: @@ -188,9 +217,9 @@ build distinct package versions. In order to produce a uniquely identifiable distribution: * If the version of a package **is not** being increased, please add or increase - the [``build/number``](https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#build-number-and-string). + the [``build/number``](https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#build-number-and-string). * If the version of a package **is** being increased, please remember to return - the [``build/number``](https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#build-number-and-string) + the [``build/number``](https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#build-number-and-string) back to 0. Feedstock Maintainers