cpython: Add native support for CMake's builtin FindPython(3)

recipes/cpython/all/conanfile.py

@@ -6,7 +6,7 @@
 from conan.errors import ConanInvalidConfiguration, ConanException
 from conan.tools.apple import is_apple_os, fix_apple_shared_install_name
 from conan.tools.env import VirtualRunEnv
-from conan.tools.files import apply_conandata_patches, copy, export_conandata_patches, get, mkdir, replace_in_file, rm, rmdir, unzip
+from conan.tools.files import apply_conandata_patches, copy, export_conandata_patches, get, mkdir, replace_in_file, rm, rmdir, save, unzip
 from conan.tools.gnu import Autotools, AutotoolsToolchain, AutotoolsDeps
 from conan.tools.layout import basic_layout
 from conan.tools.microsoft import MSBuildDeps, MSBuildToolchain, MSBuild, is_msvc, is_msvc_static_runtime, msvc_runtime_flag, msvs_toolset
@@ -339,7 +339,7 @@
     def _solution_projects(self):
         if self.options.shared:
             solution_path = os.path.join(self.source_folder, "PCbuild", "pcbuild.sln")
             projects = set(m.group(1) for m in re.finditer('"([^"]+)\\.vcxproj"', open(solution_path).read()))
 
             def project_build(name):
                 if os.path.basename(name) in self._msvc_discarded_projects:
@@ -500,7 +500,7 @@
             name, version = get_name_version(fn)
             add = True
             if name in packages:
                 pname, pversion = get_name_version(packages[name])
                 add = Version(version) > Version(pversion)
             if add:
                 packages[name] = fn
@@ -512,6 +512,70 @@
         lib_dir_path = os.path.join(self.package_folder, self._msvc_install_subprefix, "Lib").replace("\\", "/")
         self.run(f"{interpreter_path} -c \"import compileall; compileall.compile_dir('{lib_dir_path}')\"")
 
+    def _exact_lib_name(self, folder):
+        possible_extensions = ("a", "so", "dylib", "lib")
+        for file in os.listdir(folder):
+            for extension in possible_extensions:
+                if re.match(f".*\\.{extension}", file):
+                    return file
+        raise ConanException(f"No library files found in {folder}")
+
+    @property
+    def _cmake_module_path(self):
+        if is_msvc(self):
+            # On Windows, `lib` is for Python modules, `libs` is for compiled objects.
+            # Usually CMake modules are packaged with the latter.
+            return os.path.join(self._msvc_install_subprefix, "libs", "cmake")
+        else:
+            return os.path.join("lib", "cmake")
+
+    def _write_cmake_findpython_wrapper_file(self):
+        template = textwrap.dedent("""
+        if (DEFINED Python3_VERSION_STRING)
+            set(_CONAN_PYTHON_SUFFIX "3")
+        else()
+            set(_CONAN_PYTHON_SUFFIX "")
+        endif()
+        set(Python${{_CONAN_PYTHON_SUFFIX}}_EXECUTABLE {})
+        set(Python${{_CONAN_PYTHON_SUFFIX}}_LIBRARY {})
+
+        # Fails if these are set beforehand
+        unset(Python${{_CONAN_PYTHON_SUFFIX}}_INCLUDE_DIRS)
+        unset(Python${{_CONAN_PYTHON_SUFFIX}}_INCLUDE_DIR)
+
+        include(${{CMAKE_ROOT}}/Modules/FindPython${{_CONAN_PYTHON_SUFFIX}}.cmake)
+
+        # Sanity check: The former comes from FindPython(3), the latter comes from the injected find module
+        if(NOT Python${{_CONAN_PYTHON_SUFFIX}}_VERSION STREQUAL Python${{_CONAN_PYTHON_SUFFIX}}_VERSION_STRING)
+            message(FATAL_ERROR "CMake detected wrong cpython version - this is likely a bug with the cpython Conan package")
+        endif()
+
+        if (TARGET Python${{_CONAN_PYTHON_SUFFIX}}::Module)
+            target_link_libraries(Python${{_CONAN_PYTHON_SUFFIX}}::Module INTERFACE cpython::python)
+        endif()
+        if (TARGET Python${{_CONAN_PYTHON_SUFFIX}}::SABIModule)
+            target_link_libraries(Python${{_CONAN_PYTHON_SUFFIX}}::SABIModule INTERFACE cpython::python)
+        endif()
+        if (TARGET Python${{_CONAN_PYTHON_SUFFIX}}::Python)
+            target_link_libraries(Python${{_CONAN_PYTHON_SUFFIX}}::Python INTERFACE cpython::embed)
+        endif()
+        """)
+
+        # In order for the package to be relocatable, these variables must be relative to the installed CMake file
+        if is_msvc(self):
+            lib_folder = os.path.join(self.package_folder, self._msvc_install_subprefix, "libs")
+            lib_file = self._exact_lib_name(lib_folder)
+            python_exe = "${CMAKE_CURRENT_LIST_DIR}/../../" + self._cpython_interpreter_name
+            python_library = "${CMAKE_CURRENT_LIST_DIR}/../" + lib_file
+        else:
+            lib_folder = os.path.join(self.package_folder, "lib")
+            lib_file = self._exact_lib_name(lib_folder)
+            python_exe = "${CMAKE_CURRENT_LIST_DIR}/../../bin/" + self._cpython_interpreter_name
+            python_library = "${CMAKE_CURRENT_LIST_DIR}/../" + lib_file
+
+        cmake_file = os.path.join(self.package_folder, self._cmake_module_path, "use_conan_python.cmake")
+        save(self, cmake_file, template.format(python_exe, python_library))
+
     def package(self):
         copy(self, "LICENSE", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses"))
         if is_msvc(self):
@@ -522,6 +586,9 @@
             rm(self, "vcruntime*", os.path.join(self.package_folder, "bin"), recursive=True)
         else:
             autotools = Autotools(self)
+            if is_apple_os(self):
+                # FIXME: See https://github.com/python/cpython/issues/109796, this workaround is mentioned there
+                autotools.make(target="sharedinstall", args=["DESTDIR="])
             autotools.install(args=["DESTDIR="])
             rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig"))
             rmdir(self, os.path.join(self.package_folder, "share"))
@@ -556,6 +623,8 @@
                 os.symlink(f"python{self._version_suffix}", self._cpython_symlink)
         fix_apple_shared_install_name(self)
 
+        self._write_cmake_findpython_wrapper_file()
+
     @property
     def _cpython_symlink(self):
         symlink = os.path.join(self.package_folder, "bin", "python")
@@ -600,10 +669,6 @@
         return f"python{self._version_suffix}{lib_ext}"
 
     def package_info(self):
-        # FIXME: conan components Python::Interpreter component, need a target type
-        # self.cpp_info.names["cmake_find_package"] = "Python"
-        # self.cpp_info.names["cmake_find_package_multi"] = "Python"
-
         py_version = Version(self.version)
         # python component: "Build a C extension for Python"
         if is_msvc(self):
@@ -647,6 +712,12 @@
         )
         self.cpp_info.components["embed"].requires = ["python"]
 
+        # Transparent integration with CMake's FindPython(3)
+        self.cpp_info.set_property("cmake_file_name", "Python3")
+        self.cpp_info.set_property("cmake_module_file_name", "Python")
+        self.cpp_info.set_property("cmake_find_mode", "both")
+        self.cpp_info.set_property("cmake_build_modules", [os.path.join(self._cmake_module_path, "use_conan_python.cmake")])
+
         if self._supports_modules:
             # hidden components: the C extensions of python are built as dynamically loaded shared libraries.
             # C extensions or applications with an embedded Python should not need to link to them..

recipes/cpython/all/test_package/CMakeLists.txt

@@ -1,95 +1,18 @@
 cmake_minimum_required(VERSION 3.15)
 project(test_package C)
 
-find_package(cpython REQUIRED CONFIG)
+find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module Development.Embed)
 
-# FIXME: We can't modify CMake's FindPython to link dependencies pulled by
-# Conan, so here we just include them globally. This is mainly necessary for
-# MacOS missing crypt.h, which is available at configure time (in the main recipe)
-# but otherwise not at build time (in consumer packages).
-link_libraries(cpython::python)
-
-set(PY_VERSION_MAJOR_MINOR "" CACHE STRING "MAJOR.MINOR version of python")
-set(PY_VERSION "" CACHE STRING "Required version of python")
-set(PY_VERSION_SUFFIX "" CACHE STRING "Suffix of python")
-
-set(Python_ADDITIONAL_VERSIONS ${PY_VERSION}${PY_VERSION_SUFFIX} ${PY_VERSION_MAJOR_MINOR}${PY_VERSION_SUFFIX} 3${PY_VERSION_SUFFIX} ${PY_VERSION} ${PY_VERSION_MAJOR_MINOR} 3)
-message("Using Python_ADDITIONAL_VERSIONS: ${Python_ADDITIONAL_VERSIONS}")
-
-find_package(PythonInterp REQUIRED)
-find_package(PythonLibs REQUIRED)
-
-string(FIND "${PYTHON_EXECUTABLE}" "${CONAN_CPYTHON_ROOT}" ROOT_SUBPOS)
-if(ROOT_SUBPOS EQUAL -1)
-    message(FATAL_ERROR "found wrong python interpreter: ${PYTHON_EXECUTABLE}")
-endif()
-
-message(STATUS "FindPythonInterp:")
-message(STATUS "PYTHON_VERSION_STRING: ${PYTHON_VERSION_STRING}")
-message(STATUS "PYTHON_VERSION_MINOR: ${PYTHON_VERSION_MINOR}")
-message(STATUS "PYTHON_VERSION_PATCH: ${PYTHON_VERSION_PATCH}")
-message(STATUS "=============================================")
-message(STATUS "FindPythonLibs:")
-message(STATUS "PYTHON_LIBRARIES: ${PYTHON_LIBRARIES}")
-message(STATUS "PYTHON_INCLUDE_PATH: ${PYTHON_INCLUDE_PATH} (deprecated)")
-message(STATUS "PYTHON_INCLUDE_DIRS: ${PYTHON_INCLUDE_DIRS}")
-message(STATUS "PYTHON_DEBUG_LIBRARIES: ${PYTHON_DEBUG_LIBRARIES} (deprecated)")
-message(STATUS "PYTHONLIBS_VERSION_STRING: ${PYTHONLIBS_VERSION_STRING}")
-
-if(NOT PYTHON_VERSION_STRING AND NOT PYTHONLIBS_VERSION_STRING)
-    message(FATAL_ERROR "Version of python interpreter and libraries not found")
-endif()
-
-if(PYTHON_VERSION_STRING)
-    if(NOT PYTHON_VERSION_STRING VERSION_EQUAL "${PY_VERSION}")
-        message("PYTHON_VERSION_STRING does not match PY_VERSION")
-        message(FATAL_ERROR "CMake detected wrong cpython version")
-    endif()
-endif()
-
-if(PYTHONLIBS_VERSION_STRING)
-    if(NOT PYTHONLIBS_VERSION_STRING STREQUAL "${PY_VERSION}")
-        message("PYTHONLIBS_VERSION_STRING does not match PY_VERSION")
-        message(FATAL_ERROR "CMake detected wrong cpython version")
-    endif()
-endif()
+message("Python3_EXECUTABLE: ${Python3_EXECUTABLE}")
+message("Python3_INTERPRETER_ID: ${Python3_INTERPRETER_ID}")
+message("Python3_VERSION: ${Python3_VERSION}")
+message("Python3_INCLUDE_DIRS: ${Python3_INCLUDE_DIRS}")
+message("Python3_LIBRARIES: ${Python3_LIBRARIES}")
 
 option(BUILD_MODULE "Build python module")
-
 if(BUILD_MODULE)
-    add_library(spam MODULE "test_module.c")
-    target_include_directories(spam
-        PRIVATE
-            ${PYTHON_INCLUDE_DIRS}
-    )
-    target_link_libraries(spam PRIVATE
-        ${PYTHON_LIBRARIES}
-    )
-    set_property(TARGET spam PROPERTY PREFIX "")
-    if(MSVC)
-        set_target_properties(spam PROPERTIES 
-            DEBUG_POSTFIX "_d"
-            SUFFIX ".pyd"
-        )
-    endif()
-
-    option(USE_FINDPYTHON_X "Use new-style FindPythonX module")
-    if(USE_FINDPYTHON_X AND NOT CMAKE_VERSION VERSION_LESS "3.16")
-        # Require CMake 3.16 because this version introduces Python3_FIND_ABI
-        find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
-        message("Python3_EXECUTABLE: ${Python3_EXECUTABLE}")
-        message("Python3_INTERPRETER_ID: ${Python3_INTERPRETER_ID}")
-        message("Python3_VERSION: ${Python3_VERSION}")
-        message("Python3_INCLUDE_DIRS: ${Python3_INCLUDE_DIRS}")
-        message("Python3_LIBRARIES: ${Python3_LIBRARIES}")
-        if(NOT Python3_VERSION STREQUAL "${PY_VERSION}")
-            message("Python_ADDITIONAL_VERSIONS does not match PY_VERSION")
-            message(FATAL_ERROR "CMake detected wrong cpython version")
-        endif()
-
-        python3_add_library(spam2 "test_module.c")
-    endif()
+    python3_add_library(spam "test_module.c")
 endif()
 
 add_executable(${PROJECT_NAME} "test_package.c")
-target_link_libraries(${PROJECT_NAME} PRIVATE cpython::embed)
+target_link_libraries(${PROJECT_NAME} PRIVATE Python3::Python)

recipes/cpython/all/test_package/conanfile.py

@@ -25,7 +25,7 @@ def build_requirements(self):
         # The interesting problem that arises here is if you have CMake installed
         # with your global pip, then it will fail to run in this test package.
         # To avoid that, just add a requirement on CMake.
-        self.tool_requires("cmake/[>=3.15 <4]")
+        self.tool_requires("cmake/[>=3.16 <4]")
 
     def layout(self):
         cmake_layout(self)
@@ -53,30 +53,13 @@ def _py_version(self):
         else:
             return Version(self.deps_cpp_info["cpython"].version)
 
-    @property
-    def _cmake_try_FindPythonX(self):
-        return not is_msvc(self) or self.settings.build_type != "Debug"
-
     @property
     def _supports_modules(self):
         return not is_msvc(self) or self._cpython_option("shared")
 
     def generate(self):
         tc = CMakeToolchain(self)
-        version = self._py_version
         tc.cache_variables["BUILD_MODULE"] = self._supports_modules
-        tc.cache_variables["PY_VERSION_MAJOR_MINOR"] = f"{version.major}.{version.minor}"
-        tc.cache_variables["PY_VERSION"] = str(self._py_version)
-        tc.cache_variables["PY_VERSION_SUFFIX"] = "d" if self.settings.build_type == "Debug" else ""
-        tc.cache_variables["PYTHON_EXECUTABLE"] = self._python
-        tc.cache_variables["USE_FINDPYTHON_X"] = self._cmake_try_FindPythonX
-        tc.cache_variables["Python3_EXECUTABLE"] = self._python
-        tc.cache_variables["Python3_ROOT_DIR"] = self.dependencies["cpython"].package_folder
-        tc.cache_variables["Python3_USE_STATIC_LIBS"] = not self.dependencies["cpython"].options.shared
-        tc.cache_variables["Python3_FIND_FRAMEWORK"] = "NEVER"
-        tc.cache_variables["Python3_FIND_REGISTRY"] = "NEVER"
-        tc.cache_variables["Python3_FIND_IMPLEMENTATIONS"] = "CPython"
-        tc.cache_variables["Python3_FIND_STRATEGY"] = "LOCATION"
         tc.generate()
 
         deps = CMakeDeps(self)