(#23394) cpython: Add native support for CMake's builtin FindPython(3)

* Add native support for CMake's builtin FindPython(3)

* Misc cleanup

* Add Mac install workaround

* Fix FindPython on Windows

* Small cleanup

* Add review suggestions

* Cleanup self._lib_name

* Fix MSVC debug extension building

* Fix _exact_lib_name on non-Windows

* Fix hook warning

* Fix patch version

recipes/cpython/all/conanfile.py

@@ -648,6 +648,72 @@ def _msvc_package_copy(self):
         lib_dir_path = os.path.join(self.package_folder, self._msvc_install_subprefix, "Lib").replace("\\", "/")
         self.run(f"{interpreter_path} -c \"import compileall; compileall.compile_dir('{lib_dir_path}')\"")
 
+    @property
+    def _exact_lib_name(self):
+        prefix = "" if self.settings.os == "Windows" else "lib"
+        if self.settings.os == "Windows":
+            extension = "lib"
+        elif not self.options.shared:
+            extension = "a"
+        elif is_apple_os(self):
+            extension = "dylib"
+        else:
+            extension = "so"
+        return f"{prefix}{self._lib_name}.{extension}"
+
+    @property
+    def _cmake_module_path(self):
+        if is_msvc(self):
+            # On Windows, `lib` is for Python modules, `libs` is for compiled objects.
+            # Usually CMake modules are packaged with the latter.
+            return os.path.join(self._msvc_install_subprefix, "libs", "cmake")
+        else:
+            return os.path.join("lib", "cmake")
+
+    def _write_cmake_findpython_wrapper_file(self):
+        template = textwrap.dedent("""
+        if (DEFINED Python3_VERSION_STRING)
+            set(_CONAN_PYTHON_SUFFIX "3")
+        else()
+            set(_CONAN_PYTHON_SUFFIX "")
+        endif()
+        set(Python${_CONAN_PYTHON_SUFFIX}_EXECUTABLE @PYTHON_EXECUTABLE@)
+        set(Python${_CONAN_PYTHON_SUFFIX}_LIBRARY @PYTHON_LIBRARY@)
+
+        # Fails if these are set beforehand
+        unset(Python${_CONAN_PYTHON_SUFFIX}_INCLUDE_DIRS)
+        unset(Python${_CONAN_PYTHON_SUFFIX}_INCLUDE_DIR)
+
+        include(${CMAKE_ROOT}/Modules/FindPython${_CONAN_PYTHON_SUFFIX}.cmake)
+
+        # Sanity check: The former comes from FindPython(3), the latter comes from the injected find module
+        if(NOT Python${_CONAN_PYTHON_SUFFIX}_VERSION STREQUAL Python${_CONAN_PYTHON_SUFFIX}_VERSION_STRING)
+            message(FATAL_ERROR "CMake detected wrong cpython version - this is likely a bug with the cpython Conan package")
+        endif()
+
+        if (TARGET Python${_CONAN_PYTHON_SUFFIX}::Module)
+            set_target_properties(Python${_CONAN_PYTHON_SUFFIX}::Module PROPERTIES INTERFACE_LINK_LIBRARIES cpython::python)
+        endif()
+        if (TARGET Python${_CONAN_PYTHON_SUFFIX}::SABIModule)
+            set_target_properties(Python${_CONAN_PYTHON_SUFFIX}::SABIModule PROPERTIES INTERFACE_LINK_LIBRARIES cpython::python)
+        endif()
+        if (TARGET Python${_CONAN_PYTHON_SUFFIX}::Python)
+            set_target_properties(Python${_CONAN_PYTHON_SUFFIX}::Python PROPERTIES INTERFACE_LINK_LIBRARIES cpython::embed)
+        endif()
+        """)
+
+        # In order for the package to be relocatable, these variables must be relative to the installed CMake file
+        if is_msvc(self):
+            python_exe = "${CMAKE_CURRENT_LIST_DIR}/../../" + self._cpython_interpreter_name
+            python_library = "${CMAKE_CURRENT_LIST_DIR}/../" + self._exact_lib_name
+        else:
+            python_exe = "${CMAKE_CURRENT_LIST_DIR}/../../bin/" + self._cpython_interpreter_name
+            python_library = "${CMAKE_CURRENT_LIST_DIR}/../" + self._exact_lib_name
+
+        cmake_file = os.path.join(self.package_folder, self._cmake_module_path, "use_conan_python.cmake")
+        content = template.replace("@PYTHON_EXECUTABLE@", python_exe).replace("@PYTHON_LIBRARY@", python_library)
+        save(self, cmake_file, content)
+
     def package(self):
         copy(self, "LICENSE", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses"))
         if is_msvc(self):
@@ -695,6 +761,8 @@ def package(self):
                 os.symlink(f"python{self._version_suffix}", self._cpython_symlink)
         fix_apple_shared_install_name(self)
 
+        self._write_cmake_findpython_wrapper_file()
+
     @property
     def _cpython_symlink(self):
         symlink = os.path.join(self.package_folder, "bin", "python")
@@ -733,16 +801,10 @@ def _lib_name(self):
             else:
                 lib_ext = ""
         else:
-            lib_ext = self._abi_suffix + (
-                ".dll.a" if self.options.shared and self.settings.os == "Windows" else ""
-            )
+            lib_ext = self._abi_suffix
         return f"python{self._version_suffix}{lib_ext}"
 
     def package_info(self):
-        # FIXME: conan components Python::Interpreter component, need a target type
-        # self.cpp_info.names["cmake_find_package"] = "Python"
-        # self.cpp_info.names["cmake_find_package_multi"] = "Python"
-
         py_version = Version(self.version)
         # python component: "Build a C extension for Python"
         if is_msvc(self):
@@ -786,6 +848,13 @@ def package_info(self):
         )
         self.cpp_info.components["embed"].requires = ["python"]
 
+        # Transparent integration with CMake's FindPython(3)
+        self.cpp_info.set_property("cmake_file_name", "Python3")
+        self.cpp_info.set_property("cmake_module_file_name", "Python")
+        self.cpp_info.set_property("cmake_find_mode", "both")
+        self.cpp_info.set_property("cmake_build_modules", [os.path.join(self._cmake_module_path, "use_conan_python.cmake")])
+        self.cpp_info.builddirs = [self._cmake_module_path]
+
         if self._supports_modules:
             # hidden components: the C extensions of python are built as dynamically loaded shared libraries.
             # C extensions or applications with an embedded Python should not need to link to them..

recipes/cpython/all/test_package/CMakeLists.txt

@@ -1,95 +1,21 @@
 cmake_minimum_required(VERSION 3.15)
 project(test_package C)
 
-find_package(cpython REQUIRED CONFIG)
+find_package(Python3 REQUIRED COMPONENTS Interpreter Development.Module Development.Embed)
 
-# FIXME: We can't modify CMake's FindPython to link dependencies pulled by
-# Conan, so here we just include them globally. This is mainly necessary for
-# MacOS missing crypt.h, which is available at configure time (in the main recipe)
-# but otherwise not at build time (in consumer packages).
-link_libraries(cpython::python)
-
-set(PY_VERSION_MAJOR_MINOR "" CACHE STRING "MAJOR.MINOR version of python")
-set(PY_VERSION "" CACHE STRING "Required version of python")
-set(PY_VERSION_SUFFIX "" CACHE STRING "Suffix of python")
-
-set(Python_ADDITIONAL_VERSIONS ${PY_VERSION}${PY_VERSION_SUFFIX} ${PY_VERSION_MAJOR_MINOR}${PY_VERSION_SUFFIX} 3${PY_VERSION_SUFFIX} ${PY_VERSION} ${PY_VERSION_MAJOR_MINOR} 3)
-message("Using Python_ADDITIONAL_VERSIONS: ${Python_ADDITIONAL_VERSIONS}")
-
-find_package(PythonInterp REQUIRED)
-find_package(PythonLibs REQUIRED)
-
-string(FIND "${PYTHON_EXECUTABLE}" "${CONAN_CPYTHON_ROOT}" ROOT_SUBPOS)
-if(ROOT_SUBPOS EQUAL -1)
-    message(FATAL_ERROR "found wrong python interpreter: ${PYTHON_EXECUTABLE}")
-endif()
-
-message(STATUS "FindPythonInterp:")
-message(STATUS "PYTHON_VERSION_STRING: ${PYTHON_VERSION_STRING}")
-message(STATUS "PYTHON_VERSION_MINOR: ${PYTHON_VERSION_MINOR}")
-message(STATUS "PYTHON_VERSION_PATCH: ${PYTHON_VERSION_PATCH}")
-message(STATUS "=============================================")
-message(STATUS "FindPythonLibs:")
-message(STATUS "PYTHON_LIBRARIES: ${PYTHON_LIBRARIES}")
-message(STATUS "PYTHON_INCLUDE_PATH: ${PYTHON_INCLUDE_PATH} (deprecated)")
-message(STATUS "PYTHON_INCLUDE_DIRS: ${PYTHON_INCLUDE_DIRS}")
-message(STATUS "PYTHON_DEBUG_LIBRARIES: ${PYTHON_DEBUG_LIBRARIES} (deprecated)")
-message(STATUS "PYTHONLIBS_VERSION_STRING: ${PYTHONLIBS_VERSION_STRING}")
-
-if(NOT PYTHON_VERSION_STRING AND NOT PYTHONLIBS_VERSION_STRING)
-    message(FATAL_ERROR "Version of python interpreter and libraries not found")
-endif()
-
-if(PYTHON_VERSION_STRING)
-    if(NOT PYTHON_VERSION_STRING VERSION_EQUAL "${PY_VERSION}")
-        message("PYTHON_VERSION_STRING does not match PY_VERSION")
-        message(FATAL_ERROR "CMake detected wrong cpython version")
-    endif()
-endif()
-
-if(PYTHONLIBS_VERSION_STRING)
-    if(NOT PYTHONLIBS_VERSION_STRING STREQUAL "${PY_VERSION}")
-        message("PYTHONLIBS_VERSION_STRING does not match PY_VERSION")
-        message(FATAL_ERROR "CMake detected wrong cpython version")
-    endif()
-endif()
+message("Python3_EXECUTABLE: ${Python3_EXECUTABLE}")
+message("Python3_INTERPRETER_ID: ${Python3_INTERPRETER_ID}")
+message("Python3_VERSION: ${Python3_VERSION}")
+message("Python3_INCLUDE_DIRS: ${Python3_INCLUDE_DIRS}")
+message("Python3_LIBRARIES: ${Python3_LIBRARIES}")
 
 option(BUILD_MODULE "Build python module")
-
 if(BUILD_MODULE)
-    add_library(spam MODULE "test_module.c")
-    target_include_directories(spam
-        PRIVATE
-            ${PYTHON_INCLUDE_DIRS}
-    )
-    target_link_libraries(spam PRIVATE
-        ${PYTHON_LIBRARIES}
-    )
-    set_property(TARGET spam PROPERTY PREFIX "")
+    python3_add_library(spam "test_module.c")
     if(MSVC)
-        set_target_properties(spam PROPERTIES 
-            DEBUG_POSTFIX "_d"
-            SUFFIX ".pyd"
-        )
-    endif()
-
-    option(USE_FINDPYTHON_X "Use new-style FindPythonX module")
-    if(USE_FINDPYTHON_X AND NOT CMAKE_VERSION VERSION_LESS "3.16")
-        # Require CMake 3.16 because this version introduces Python3_FIND_ABI
-        find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
-        message("Python3_EXECUTABLE: ${Python3_EXECUTABLE}")
-        message("Python3_INTERPRETER_ID: ${Python3_INTERPRETER_ID}")
-        message("Python3_VERSION: ${Python3_VERSION}")
-        message("Python3_INCLUDE_DIRS: ${Python3_INCLUDE_DIRS}")
-        message("Python3_LIBRARIES: ${Python3_LIBRARIES}")
-        if(NOT Python3_VERSION STREQUAL "${PY_VERSION}")
-            message("Python_ADDITIONAL_VERSIONS does not match PY_VERSION")
-            message(FATAL_ERROR "CMake detected wrong cpython version")
-        endif()
-
-        python3_add_library(spam2 "test_module.c")
+        set_target_properties(spam PROPERTIES DEBUG_POSTFIX "_d")
     endif()
 endif()
 
 add_executable(${PROJECT_NAME} "test_package.c")
-target_link_libraries(${PROJECT_NAME} PRIVATE cpython::embed)
+target_link_libraries(${PROJECT_NAME} PRIVATE Python3::Python)

recipes/cpython/all/test_package/conanfile.py

@@ -25,7 +25,7 @@ def build_requirements(self):
         # The interesting problem that arises here is if you have CMake installed
         # with your global pip, then it will fail to run in this test package.
         # To avoid that, just add a requirement on CMake.
-        self.tool_requires("cmake/[>=3.15 <4]")
+        self.tool_requires("cmake/[>=3.16 <4]")
 
     def layout(self):
         cmake_layout(self)
@@ -59,30 +59,13 @@ def _test_setuptools(self):
         # https://github.com/python/cpython/pull/101039
         return can_run(self) and self._supports_modules and self._py_version < "3.12"
 
-    @property
-    def _cmake_try_FindPythonX(self):
-        return not is_msvc(self) or self.settings.build_type != "Debug"
-
     @property
     def _supports_modules(self):
         return not is_msvc(self) or self._cpython_option("shared")
 
     def generate(self):
         tc = CMakeToolchain(self)
-        version = self._py_version
         tc.cache_variables["BUILD_MODULE"] = self._supports_modules
-        tc.cache_variables["PY_VERSION_MAJOR_MINOR"] = f"{version.major}.{version.minor}"
-        tc.cache_variables["PY_VERSION"] = str(self._py_version)
-        tc.cache_variables["PY_VERSION_SUFFIX"] = "d" if self.settings.build_type == "Debug" else ""
-        tc.cache_variables["PYTHON_EXECUTABLE"] = self._python
-        tc.cache_variables["USE_FINDPYTHON_X"] = self._cmake_try_FindPythonX
-        tc.cache_variables["Python3_EXECUTABLE"] = self._python
-        tc.cache_variables["Python3_ROOT_DIR"] = self.dependencies["cpython"].package_folder
-        tc.cache_variables["Python3_USE_STATIC_LIBS"] = not self.dependencies["cpython"].options.shared
-        tc.cache_variables["Python3_FIND_FRAMEWORK"] = "NEVER"
-        tc.cache_variables["Python3_FIND_REGISTRY"] = "NEVER"
-        tc.cache_variables["Python3_FIND_IMPLEMENTATIONS"] = "CPython"
-        tc.cache_variables["Python3_FIND_STRATEGY"] = "LOCATION"
         tc.generate()
 
         deps = CMakeDeps(self)