Merge pull request #2672 from nilsleiffischer/cmake_find_python

Make finding Python in build system more robust, update Docker container
sxs-collaboration · Dec 22, 2020 · dfe1c6f · dfe1c6f
2 parents 5cca433 + 6b3d1dd
commit dfe1c6f
Show file tree

Hide file tree

Showing 41 changed files with 136 additions and 942 deletions.
diff --git a/.github/workflows/Tests.yaml b/.github/workflows/Tests.yaml
@@ -418,7 +418,7 @@ jobs:
           -D USE_PCH=${USE_PCH}
           -D USE_CCACHE=ON
           -D BUILD_PYTHON_BINDINGS=${BUILD_PYTHON_BINDINGS:-'ON'}
-          -D PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-'/usr/bin/python3'}
+          -D Python_EXECUTABLE=${PYTHON_EXECUTABLE:-'/usr/bin/python3'}
           -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS:-'ON'}
           -D ASAN=${ASAN:-'OFF'}
           -D UBSAN_UNDEFINED=${UBSAN_UNDEFINED:-'OFF'}

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,7 +1,7 @@
 # Distributed under the MIT License.
 # See LICENSE.txt for details.
 
-cmake_minimum_required(VERSION 3.8.0)
+cmake_minimum_required(VERSION 3.12.0)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 
@@ -78,9 +78,6 @@ include(SetupListTargets)
 include(AddSpectreExecutable)
 include(CheckBrokenArray0)
 
-# Need to load python libs before boost.python.
-include(SetupPythonLibs)
-
 # Blaze depends on both Blas and Lapack, since LIBXSMM is a Blas
 # alternative we set it up early too.
 include(SetupBlas)
@@ -94,10 +91,8 @@ include(SetupGoogleBenchmark)
 include(SetupGsl)
 include(SetupHdf5)
 include(SetupAllocator)
-include(SetupNumPy)
 include(SetupPapi)
 include(SetupPybind11)
-include(SetupSciPy)
 include(SetupStl)
 include(SetupLibsharp)
 include(SetupYamlCpp)
@@ -127,6 +122,7 @@ include(CodeCoverageDetection)
 include(SpectreAddLibraries)
 
 enable_testing(true)
+include(SetupPypp)
 include(SpectreAddTestLibs)
 include(SpectreAddCatchTests)
 include(AddInputFileTests)

diff --git a/cmake/AddInputFileTests.cmake b/cmake/AddInputFileTests.cmake
@@ -5,7 +5,7 @@ option(SPECTRE_INPUT_FILE_TEST_TIMEOUT_FACTOR
   "Multiply timeout for input file tests by this factor"
   1)
 
-find_package(PythonInterp REQUIRED)
+find_package(Python REQUIRED)
 
 function(add_single_input_file_test INPUT_FILE EXECUTABLE CHECK_TYPE TIMEOUT)
   # Extract just the name of the input file
@@ -163,10 +163,10 @@ file(
   ${PROJECT_BINARY_DIR}/tmp/InputFileExecuteAndClean.sh
   "\
 #!/bin/sh\n\
-${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/CleanOutput.py -v --force \
+${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/CleanOutput.py -v --force \
 --input-file $2 --output-dir ${CMAKE_BINARY_DIR}
 ${CMAKE_BINARY_DIR}/bin/$1 --input-file $2 && \
-${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/CleanOutput.py -v \
+${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/CleanOutput.py -v \
 --input-file $2 --output-dir ${CMAKE_BINARY_DIR}\n"
 )
 

diff --git a/cmake/FindNumPy.cmake b/cmake/FindNumPy.cmake
@@ -1,9 +1,9 @@
 # Distributed under the MIT License.
 # See LICENSE.txt for details.
 
-find_package(PythonInterp REQUIRED)
+find_package(Python REQUIRED)
 
-execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
+execute_process(COMMAND "${Python_EXECUTABLE}" "-c"
   "import numpy as n; print(n.__version__); print(n.get_include());"
   RESULT_VARIABLE RESULT
   OUTPUT_VARIABLE OUTPUT

diff --git a/cmake/FindPythonModule.cmake b/cmake/FindPythonModule.cmake
@@ -15,7 +15,7 @@ function(find_python_module module is_required)
     endif()
     # A module's location is usually a directory, but for binary modules
     # it's a .so file.
-    execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c"
+    execute_process(COMMAND "${Python_EXECUTABLE}" "-c"
         "import re, ${module}; print(re.compile('/__init__.py.*').sub('',${module}.__file__))"
         RESULT_VARIABLE _${module}_status
         OUTPUT_VARIABLE _${module}_location

diff --git a/cmake/FindSciPy.cmake b/cmake/FindSciPy.cmake
diff --git a/cmake/Findpybind11.cmake b/cmake/Findpybind11.cmake
diff --git a/cmake/PrintUsefulCMakeInfo.cmake b/cmake/PrintUsefulCMakeInfo.cmake
@@ -21,9 +21,9 @@ message(STATUS "ASAN: ${ASAN}")
 message(STATUS "UBSAN_UNDEFINED: ${UBSAN_UNDEFINED}")
 message(STATUS "UBSAN_INTEGER: ${UBSAN_INTEGER}")
 message(STATUS "USE_SYSTEM_INCLUDE: ${USE_SYSTEM_INCLUDE}")
-if (PYTHONINTERP_FOUND)
-  message(STATUS "Python: " ${PYTHON_EXECUTABLE})
-  message(STATUS "Python Version: ${PYTHON_VERSION_STRING}")
+if (Python_FOUND)
+  message(STATUS "Python: " ${Python_EXECUTABLE})
+  message(STATUS "Python Version: ${Python_VERSION}")
 else()
   message(STATUS "Python: Not found")
 endif()

diff --git a/cmake/SetupCharmModuleFunctions.cmake b/cmake/SetupCharmModuleFunctions.cmake
@@ -1,8 +1,6 @@
 # Distributed under the MIT License.
 # See LICENSE.txt for details.
 
-find_package(PythonInterp REQUIRED)
-
 # Add a function to generate the charm interface files for the module.
 function(add_charm_module MODULE)
   # Arguments:

diff --git a/cmake/SetupDoxygen.cmake b/cmake/SetupDoxygen.cmake
@@ -31,8 +31,8 @@ if (DOXYGEN_FOUND)
     )
 
   # We need Python for postprocessing the documentation
-  find_package(PythonInterp)
-  if (PYTHONINTERP_FOUND)
+  find_package(Python)
+  if (Python_FOUND)
     include(FindPythonModule)
     find_python_module(bs4 FALSE)
     find_python_module(pybtex FALSE)
@@ -41,7 +41,7 @@ if (DOXYGEN_FOUND)
       # output
       set(
         DOCS_POST_PROCESS_COMMAND
-        "${PYTHON_EXECUTABLE} \
+        "${Python_EXECUTABLE} \
 ${CMAKE_SOURCE_DIR}/docs/config/postprocess_docs.py \
 --html-dir ${PROJECT_BINARY_DIR}/docs/html \
 --references-file ${CMAKE_SOURCE_DIR}/docs/References.bib"
@@ -71,10 +71,10 @@ ${DOCS_POST_PROCESS_COMMAND} && exit \${generate_docs_exit}\n"
         message(WARNING "Pybtex missing. Install with: pip install pybtex")
       endif()
     endif (PY_BS4 AND PY_PYBTEX)
-  else (PYTHONINTERP_FOUND)
+  else (Python_FOUND)
     message(WARNING "Doxygen documentation postprocessing is disabled because a"
     " Python interpreter was not found.")
-  endif (PYTHONINTERP_FOUND)
+  endif (Python_FOUND)
 
   # Parse the command into a CMake list for the `add_custom_target`
   separate_arguments(GENERATE_DOCS_COMMAND)
@@ -130,7 +130,7 @@ ${DOCS_POST_PROCESS_COMMAND} && exit \${generate_docs_exit}\n"
     add_custom_target(
       doc-coverage
 
-      COMMAND ${PYTHON_EXECUTABLE}
+      COMMAND ${Python_EXECUTABLE}
       -m coverxygen
       --xml-dir ${CMAKE_BINARY_DIR}/docs/xml
       --src-dir ${CMAKE_SOURCE_DIR}

diff --git a/cmake/SetupGitHooks.cmake b/cmake/SetupGitHooks.cmake
@@ -19,7 +19,7 @@ if(USE_GIT_HOOKS)
 
   # The logic is inverted because shell
   if(NOT CHECK_SOURCE_DIR_WRITABLE_RESULT AND EXISTS ${CMAKE_SOURCE_DIR}/.git)
-    find_package(PythonInterp REQUIRED)
+    find_package(Python REQUIRED)
 
     # We use several client-side git hooks to ensure commits are correct as
     # early as possible.

diff --git a/cmake/SetupNumPy.cmake b/cmake/SetupNumPy.cmake
diff --git a/cmake/SetupPybind11.cmake b/cmake/SetupPybind11.cmake
@@ -4,22 +4,26 @@
 option(BUILD_PYTHON_BINDINGS "Build the python bindings for SpECTRE" OFF)
 
 if(BUILD_PYTHON_BINDINGS)
-  # Make sure to find the Python interpreter first, so it is consistent with
-  # the one that pybind11 uses
-  find_package(PythonInterp)
-  find_package(PythonLibs)
-  # Uses `Findpybind11.cmake` to find the headers. Since we can't rely on the
-  # corresponding cmake files to be installed as well we bundle them in
-  # `external/pybind11`.
-  find_package(pybind11 REQUIRED)
+  # Make sure to find Python first so it's consistent with pybind11
+  find_package(Python COMPONENTS Interpreter Development)
 
-  # Load the CMake files from `external/pybind11`
-  list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/external/pybind11/tools")
-  include(pybind11Tools)
+  # Try to find the pybind11-config tool to find pybind11's CMake config files
+  find_program(PYBIND11_CONFIG_TOOL pybind11-config)
+  set(PYBIND11_CMAKEDIR "")
+  if(PYBIND11_CONFIG_TOOL)
+    execute_process(
+      COMMAND "${PYBIND11_CONFIG_TOOL}" "--cmakedir"
+      OUTPUT_VARIABLE PYBIND11_CMAKEDIR
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    message(STATUS "Found pybind11-config tool (${PYBIND11_CONFIG_TOOL}) and "
+      "determined CMake dir: ${PYBIND11_CMAKEDIR}")
+  endif()
+
+  find_package(pybind11 2.6.0 REQUIRED HINTS "${PYBIND11_CMAKEDIR}")
 
   set_property(
     GLOBAL APPEND PROPERTY SPECTRE_THIRD_PARTY_LIBS
-    pybind11
+    pybind11::headers
     )
 
   message(STATUS "Pybind11 include: ${pybind11_INCLUDE_DIR}")

diff --git a/cmake/SetupPypp.cmake b/cmake/SetupPypp.cmake
@@ -0,0 +1,30 @@
+# Distributed under the MIT License.
+# See LICENSE.txt for details.
+
+# To run Python tests we need the development component (libs and include dirs).
+# It provides the `Python::Python` imported target.
+find_package(Python REQUIRED COMPONENTS Development)
+
+# We can't rely on CMake 3.14 quite yet so we backport the `Python::NumPy`
+# imported target that FindPython in CMake 3.14+ could also provide.
+find_package(NumPy 1.10 REQUIRED)
+
+message(STATUS "NumPy incl: " ${NUMPY_INCLUDE_DIRS})
+message(STATUS "NumPy vers: " ${NUMPY_VERSION})
+
+file(APPEND
+  "${CMAKE_BINARY_DIR}/LibraryVersions.txt"
+  "NumPy Version: ${NUMPY_VERSION}\n"
+  )
+
+# Also check that SciPy is installed
+find_python_module(scipy TRUE)
+
+add_library(Python::NumPy INTERFACE IMPORTED)
+set_property(TARGET Python::NumPy PROPERTY
+  INTERFACE_INCLUDE_DIRECTORIES ${NUMPY_INCLUDE_DIRS})
+
+set_property(
+  GLOBAL APPEND PROPERTY SPECTRE_THIRD_PARTY_LIBS
+  Python::NumPy Python::Python
+  )
diff --git a/cmake/SetupPythonLibs.cmake b/cmake/SetupPythonLibs.cmake
diff --git a/cmake/SetupSciPy.cmake b/cmake/SetupSciPy.cmake
diff --git a/cmake/SetupVersionFile.cmake b/cmake/SetupVersionFile.cmake
@@ -30,5 +30,5 @@ file(WRITE
   "CMAKE_C_COMPILER_VERSION: ${CMAKE_C_COMPILER_VERSION}\n"
   "CMAKE_Fortran_COMPILER: ${CMAKE_Fortran_COMPILER}\n"
   "CMAKE_Fortran_COMPILER_VERSION: ${CMAKE_Fortran_COMPILER_VERSION}\n"
-  "Python Version: ${PYTHON_VERSION_STRING}\n"
+  "Python Version: ${Python_VERSION}\n"
   )