fix numpy2.0/2.1 bug in release/2.6 (#67958)

* 升级编译所需要的依赖 * cherry-pick-np2.0 * fix * fix * fix * fix * fix * fix * fix --------- Co-authored-by: XieYunshen <1084314248@qq.com>
PaddlePaddle · Sep 5, 2024 · d16d420 · d16d420
1 parent 64c6937
commit d16d420
Show file tree

Hide file tree

Showing 34 changed files with 207 additions and 175 deletions.
diff --git a/cmake/external/pybind11.cmake b/cmake/external/pybind11.cmake
@@ -17,23 +17,26 @@ include(ExternalProject)
 set(PYBIND_PREFIX_DIR ${THIRD_PARTY_PATH}/pybind)
 set(PYBIND_SOURCE_DIR ${PYBIND_PREFIX_DIR}/src/extern_pybind)
 set(PYBIND_INCLUDE_DIR ${PYBIND_SOURCE_DIR}/include)
-set(PYBIND_TAG v2.10.3)
 set(SOURCE_DIR ${PADDLE_SOURCE_DIR}/third_party/pybind)
 set(SOURCE_INCLUDE_DIR ${SOURCE_DIR}/include)
 
 include_directories(${PYBIND_INCLUDE_DIR})
 
+# It can be safely removed in gcc9.1+
 set(PYBIND_PATCH_COMMAND "")
-if(NOT WIN32)
-  file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/pybind/cast.h.patch
-       native_dst)
+if(LINUX
+   AND (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+   AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)
+  set(PYBIND_TAG v2.12.0)
+  file(TO_NATIVE_PATH
+       ${PADDLE_SOURCE_DIR}/patches/pybind/detail/internals.h.patch native_dst)
   # Note: [Why calling some `git` commands before `patch`?]
-  # Paddle's CI uses cache to accelarate the make process. However, error might raise when patch codes in two scenarios:
+  # Paddle's CI uses cache to accelerate the make process. However, error might raise when patch codes in two scenarios:
   # 1. Patch to the wrong version: the tag version of CI's cache falls behind PYBIND_TAG, use `git checkout ${PYBIND_TAG}` to solve this.
   # 2. Patch twice: the tag version of cache == PYBIND_TAG, but patch has already applied to cache.
   set(PYBIND_PATCH_COMMAND
       git checkout -- . && git checkout ${PYBIND_TAG} && patch -Nd
-      ${SOURCE_INCLUDE_DIR}/pybind11 < ${native_dst})
+      ${SOURCE_INCLUDE_DIR}/pybind11/detail < ${native_dst})
 endif()
 
 ExternalProject_Add(

diff --git a/paddle/fluid/pybind/tensor.cc b/paddle/fluid/pybind/tensor.cc
@@ -215,8 +215,8 @@ void BindTensor(pybind11::module &m) {  // NOLINT
           // TensorToPyArray() according to the dtype and copy
           // parameters.
           "__array__",
-          [](phi::DenseTensor &self, py::object dtype, py::object copy) {
-            return TensorToPyArray(self);
+          [](phi::DenseTensor &self, py::object dtype, py::object copy) { //NOLINT
+            return TensorToPyArray(self,copy);
           },
           py::arg("dtype") = py::none(),
           py::arg("copy") = py::none())

diff --git a/paddle/fluid/pybind/tensor_py.h b/paddle/fluid/pybind/tensor_py.h
@@ -955,8 +955,7 @@ inline phi::DenseTensor *PySliceTensor(const phi::DenseTensor &self,
   }
 }
 
-inline py::array TensorToPyArray(const phi::DenseTensor &tensor,
-                                 bool need_deep_copy = false) {
+inline py::array TensorToPyArray(const phi::DenseTensor &tensor,py::object copy=py::none()) {
   if (!tensor.IsInitialized()) {
     return py::array();
   }
@@ -984,9 +983,8 @@ inline py::array TensorToPyArray(const phi::DenseTensor &tensor,
 
   std::string py_dtype_str = details::TensorDTypeToPyDTypeStr(
       framework::TransToProtoVarType(tensor.dtype()));
-
   if (!is_gpu_tensor && !is_xpu_tensor && !is_custom_device_tensor) {
-    if (!need_deep_copy) {
+    if (!copy.is_none()&& !copy) {
       auto base = py::cast(std::move(tensor));
       return py::array(py::dtype(py_dtype_str.c_str()),
                        py_dims,

diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat
@@ -111,7 +111,7 @@ if "%WITH_PYTHON%" == "ON" (
     where pip
     python -m pip install --upgrade pip
     python -m pip install -r %work_dir%\paddle\scripts\compile_requirements.txt
-    python -m pip install -r %work_dir%\python\requirements.txt
+    python -m pip install -r %work_dir%\python\requirements.txt --no-cache-dir --force-reinstall
     if !ERRORLEVEL! NEQ 0 (
         echo pip install requirements.txt failed!
         exit /b 5

diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh
@@ -3544,7 +3544,7 @@ function run_setup(){
                 export PYTHON_EXECUTABLE=/Library/Frameworks/Python.framework/Versions/3.8/bin/python3
                 export PYTHON_INCLUDE_DIR=/Library/Frameworks/Python.framework/Versions/3.8/include/python3.8/
                 export PYTHON_LIBRARY=/Library/Frameworks/Python.framework/Versions/3.8/lib/libpython3.8.dylib
-                pip3.8 install --user -r ${PADDLE_ROOT}/python/requirements.txt
+                pip3.8 install --user -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
             else
                 exit 1
             fi
@@ -3557,7 +3557,7 @@ function run_setup(){
                 export PYTHON_EXECUTABLE=/Library/Frameworks/Python.framework/Versions/3.9/bin/python3
                 export PYTHON_INCLUDE_DIR=/Library/Frameworks/Python.framework/Versions/3.9/include/python3.9/
                 export PYTHON_LIBRARY=/Library/Frameworks/Python.framework/Versions/3.9/lib/libpython3.9.dylib
-                pip3.9 install --user -r ${PADDLE_ROOT}/python/requirements.txt
+                pip3.9 install --user -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
             else
                 exit 1
             fi
@@ -3570,7 +3570,7 @@ function run_setup(){
                 export PYTHON_EXECUTABLE=/Library/Frameworks/Python.framework/Versions/3.10/bin/python3
                 export PYTHON_INCLUDE_DIR=/Library/Frameworks/Python.framework/Versions/3.10/include/python3.10/
                 export PYTHON_LIBRARY=/Library/Frameworks/Python.framework/Versions/3.10/lib/libpython3.10.dylib
-                pip3.10 install --user -r ${PADDLE_ROOT}/python/requirements.txt
+                pip3.10 install --user -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
             else
                 exit 1
             fi
@@ -3583,7 +3583,7 @@ function run_setup(){
                 export PYTHON_EXECUTABLE=/Library/Frameworks/Python.framework/Versions/3.11/bin/python3
                 export PYTHON_INCLUDE_DIR=/Library/Frameworks/Python.framework/Versions/3.11/include/python3.11/
                 export PYTHON_LIBRARY=/Library/Frameworks/Python.framework/Versions/3.11/lib/libpython3.11.dylib
-                pip3.11 install --user -r ${PADDLE_ROOT}/python/requirements.txt
+                pip3.11 install --user -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
             else
                 exit 1
             fi
@@ -3596,7 +3596,7 @@ function run_setup(){
                 export PYTHON_EXECUTABLE=/Library/Frameworks/Python.framework/Versions/3.12/bin/python3
                 export PYTHON_INCLUDE_DIR=/Library/Frameworks/Python.framework/Versions/3.12/include/python3.12/
                 export PYTHON_LIBRARY=/Library/Frameworks/Python.framework/Versions/3.12/lib/libpython3.12.dylib
-                pip3.12 install --user -r ${PADDLE_ROOT}/python/requirements.txt
+                pip3.12 install --user -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
             else
                 exit 1
             fi
@@ -3611,7 +3611,7 @@ function run_setup(){
                 export PYTHON_EXECUTABLE=/opt/_internal/cpython-3.8.0/bin/python3.8
                 export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.8.0/include/python3.8
                 export PYTHON_LIBRARIES=/opt/_internal/cpython-3.8.0/lib/libpython3.so
-                pip3.8 install -r ${PADDLE_ROOT}/python/requirements.txt
+                pip3.8 install -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
                 pip3.8 install -r ${PADDLE_ROOT}/paddle/scripts/compile_requirements.txt
             elif [ "$1" == "cp39-cp39" ]; then
                 export LD_LIBRARY_PATH=/opt/_internal/cpython-3.9.0/lib/:${LD_LIBRARY_PATH}
@@ -3620,7 +3620,7 @@ function run_setup(){
                 export PYTHON_EXECUTABLE=/opt/_internal/cpython-3.9.0/bin/python3.9
                 export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.9.0/include/python3.9
                 export PYTHON_LIBRARIES=/opt/_internal/cpython-3.9.0/lib/libpython3.so
-                pip3.9 install -r ${PADDLE_ROOT}/python/requirements.txt
+                pip3.9 install -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
                 pip3.9 install -r ${PADDLE_ROOT}/paddle/scripts/compile_requirements.txt
             elif [ "$1" == "cp310-cp310" ]; then
                 export LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH}
@@ -3629,7 +3629,7 @@ function run_setup(){
                 export PYTHON_EXECUTABLE=/opt/_internal/cpython-3.10.0/bin/python3.10
                 export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.10.0/include/python3.10
                 export PYTHON_LIBRARIES=/opt/_internal/cpython-3.10.0/lib/libpython3.so
-                pip3.10 install -r ${PADDLE_ROOT}/python/requirements.txt
+                pip3.10 install -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
                 pip3.10 install -r ${PADDLE_ROOT}/paddle/scripts/compile_requirements.txt
             elif [ "$1" == "cp311-cp311" ]; then
                 export LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH}
@@ -3638,7 +3638,7 @@ function run_setup(){
                 export PYTHON_EXECUTABLE=/opt/_internal/cpython-3.11.0/bin/python3.11
                 export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.11.0/include/python3.11
                 export PYTHON_LIBRARIES=/opt/_internal/cpython-3.11.0/lib/libpython3.so
-                pip3.11 install -r ${PADDLE_ROOT}/python/requirements.txt
+                pip3.11 install -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
                 pip3.11 install -r ${PADDLE_ROOT}/paddle/scripts/compile_requirements.txt
             elif [ "$1" == "cp312-cp312" ]; then
                 export LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH}
@@ -3647,14 +3647,15 @@ function run_setup(){
                 export PYTHON_EXECUTABLE=/opt/_internal/cpython-3.12.0/bin/python3.12
                 export PYTHON_INCLUDE_DIR=/opt/_internal/cpython-3.12.0/include/python3.12
                 export PYTHON_LIBRARIES=/opt/_internal/cpython-3.12.0/lib/libpython3.so
-                pip3.12 install -r ${PADDLE_ROOT}/python/requirements.txt
-                pip3.12 install -r ${PADDLE_ROOT}/paddle/scripts/compile_requirements.txt
+                pip3.12 install -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
+                pip3.12 install -r ${PADDLE_ROOT}/paddle/scripts/compile_requirements.txt --no-cache-dir --force-reinstall
            fi
         else
-            pip install -r ${PADDLE_ROOT}/python/requirements.txt
+            pip install -r ${PADDLE_ROOT}/python/requirements.txt --no-cache-dir --force-reinstall
         fi
     fi
 
+
     if [ "$SYSTEM" == "Darwin" ]; then
         WITH_DISTRIBUTE="OFF"
         WITH_AVX=${WITH_AVX:-ON}

diff --git a/patches/pybind/cast.h.patch b/patches/pybind/cast.h.patch
@@ -12,4 +12,4 @@ index 3a404602..9054478c 100644
 +    return caster;
  }
  template <typename T>
- typename make_caster<T>::template cast_op_type<typename std::add_rvalue_reference<T>::type>
+ typename make_caster<T>::template cast_op_type<typename std::add_rvalue_reference<T>::type>
diff --git a/patches/pybind/detail/internals.h.patch b/patches/pybind/detail/internals.h.patch
@@ -0,0 +1,22 @@
+diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h
+index c1047e4a..e09a6495 100644
+--- a/include/pybind11/detail/internals.h
++++ b/include/pybind11/detail/internals.h
+@@ -193,11 +193,18 @@ struct internals {
+     PyTypeObject *default_metaclass;
+     PyObject *instance_base;
+ #if defined(WITH_THREAD)
++#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ <= 8
++#pragma GCC diagnostic push
++#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
++#endif
+     // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
+     PYBIND11_TLS_KEY_INIT(tstate)
+ #    if PYBIND11_INTERNALS_VERSION > 4
+     PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key)
+ #    endif // PYBIND11_INTERNALS_VERSION > 4
++#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ <= 8
++#pragma GCC diagnostic pop
++#endif
+     // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined:
+     PyInterpreterState *istate = nullptr;
diff --git a/python/paddle/hapi/callbacks.py b/python/paddle/hapi/callbacks.py
@@ -1279,10 +1279,10 @@ def _reset(self):
             self.mode == 'auto' and 'acc' not in self.monitor
         ):
             self.monitor_op = lambda a, b: np.less(a, b - self.min_delta)
-            self.best = np.Inf
+            self.best = np.inf
         else:
             self.monitor_op = lambda a, b: np.greater(a, b + self.min_delta)
-            self.best = -np.Inf
+            self.best = -np.inf
         self.cooldown_counter = 0
         self.wait = 0