From eb45dbd2034efb1784002bb83b4215a5f6d89cde Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 27 Nov 2024 15:31:30 -0800 Subject: [PATCH 1/6] Remve flags which no longer exist in stubgen 2.4.2 --- cmake/morpheus_utils/python/python_module_tools.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/morpheus_utils/python/python_module_tools.cmake b/cmake/morpheus_utils/python/python_module_tools.cmake index ac1735b..d1eb4b0 100644 --- a/cmake/morpheus_utils/python/python_module_tools.cmake +++ b/cmake/morpheus_utils/python/python_module_tools.cmake @@ -623,7 +623,7 @@ macro(__create_python_library MODULE_NAME) # Before installing, create the custom command to generate the stubs add_custom_command( OUTPUT ${module_binary_stub_file} - COMMAND ${Python3_EXECUTABLE} -m pybind11_stubgen ${TARGET_NAME} --no-setup-py --log-level WARN -o ./ --root-module-suffix \"\" + COMMAND ${Python3_EXECUTABLE} -m pybind11_stubgen ${TARGET_NAME} -o ./ --root-suffix \"\" DEPENDS ${PYTHON_ACTIVE_PACKAGE_NAME}-modules $ COMMENT "Building stub for python module ${TARGET_NAME}..." WORKING_DIRECTORY ${module_root_binary_dir} From b669e94ca6d6a7c2fa7526c81d023369d44f0620 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 27 Nov 2024 15:55:15 -0800 Subject: [PATCH 2/6] Remove --root-suffix flag, this defaults to an empty string, Add --ignore-invalid-identifiers flag avoids issue where not all symbols are loaded into the current Python env --- cmake/morpheus_utils/python/python_module_tools.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/morpheus_utils/python/python_module_tools.cmake b/cmake/morpheus_utils/python/python_module_tools.cmake index d1eb4b0..d82f51d 100644 --- a/cmake/morpheus_utils/python/python_module_tools.cmake +++ b/cmake/morpheus_utils/python/python_module_tools.cmake @@ -623,7 +623,7 @@ macro(__create_python_library MODULE_NAME) # Before installing, create the custom command to generate the stubs add_custom_command( OUTPUT ${module_binary_stub_file} - COMMAND ${Python3_EXECUTABLE} -m pybind11_stubgen ${TARGET_NAME} -o ./ --root-suffix \"\" + COMMAND ${Python3_EXECUTABLE} -m pybind11_stubgen --ignore-invalid-identifiers '.*' ${TARGET_NAME} -o ./ DEPENDS ${PYTHON_ACTIVE_PACKAGE_NAME}-modules $ COMMENT "Building stub for python module ${TARGET_NAME}..." WORKING_DIRECTORY ${module_root_binary_dir} From 417505ef105f53663d2bde87c69e16182af28170 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Mon, 2 Dec 2024 16:21:02 -0800 Subject: [PATCH 3/6] Send the pyi file to a temp dir and then copy it to the expected location --- .../python/python_module_tools.cmake | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/cmake/morpheus_utils/python/python_module_tools.cmake b/cmake/morpheus_utils/python/python_module_tools.cmake index d82f51d..85ecdf0 100644 --- a/cmake/morpheus_utils/python/python_module_tools.cmake +++ b/cmake/morpheus_utils/python/python_module_tools.cmake @@ -618,15 +618,24 @@ macro(__create_python_library MODULE_NAME) cmake_path(RELATIVE_PATH _ARGS_MODULE_ROOT BASE_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE module_root_relative) cmake_path(APPEND PROJECT_BINARY_DIR ${module_root_relative} OUTPUT_VARIABLE module_root_binary_dir) - cmake_path(APPEND module_root_binary_dir ${SOURCE_RELATIVE_PATH} ${MODULE_NAME} "__init__.pyi" OUTPUT_VARIABLE module_binary_stub_file) + cmake_path(APPEND module_root_binary_dir ${SOURCE_RELATIVE_PATH} "stubs" OUTPUT_VARIABLE module_binary_stub_temp_dir) + cmake_path(APPEND module_binary_stub_temp_dir ${SOURCE_RELATIVE_PATH} "${MODULE_NAME}.pyi" OUTPUT_VARIABLE module_binary_stub_temp_file) + cmake_path(APPEND module_root_binary_dir ${SOURCE_RELATIVE_PATH} "${MODULE_NAME}" "__init__.pyi" OUTPUT_VARIABLE module_binary_stub_file) # Before installing, create the custom command to generate the stubs add_custom_command( - OUTPUT ${module_binary_stub_file} - COMMAND ${Python3_EXECUTABLE} -m pybind11_stubgen --ignore-invalid-identifiers '.*' ${TARGET_NAME} -o ./ + OUTPUT ${module_binary_stub_temp_file} + COMMAND ${Python3_EXECUTABLE} -m pybind11_stubgen --ignore-invalid-identifiers '.*' ${TARGET_NAME} -o ${module_binary_stub_temp_dir} DEPENDS ${PYTHON_ACTIVE_PACKAGE_NAME}-modules $ COMMENT "Building stub for python module ${TARGET_NAME}..." - WORKING_DIRECTORY ${module_root_binary_dir} + WORKING_DIRECTORY ${module_binary_stub_dir} + ) + + add_custom_command( + OUTPUT ${module_binary_stub_file} + COMMAND ${CMAKE_COMMAND} -E copy ${module_binary_stub_temp_file} ${module_binary_stub_file} + DEPENDS ${module_binary_stub_temp_file} + COMMENT "Moving target ${module_binary_stub_temp_file} to ${module_binary_stub_file}" ) # Add a custom target to ensure the stub generation runs From a960e7e2a0bd4194c489b83023560541ccfbc2c2 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 3 Dec 2024 11:21:30 -0800 Subject: [PATCH 4/6] Revert the change of the working dir --- cmake/morpheus_utils/python/python_module_tools.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/morpheus_utils/python/python_module_tools.cmake b/cmake/morpheus_utils/python/python_module_tools.cmake index 85ecdf0..0ad2b81 100644 --- a/cmake/morpheus_utils/python/python_module_tools.cmake +++ b/cmake/morpheus_utils/python/python_module_tools.cmake @@ -628,7 +628,7 @@ macro(__create_python_library MODULE_NAME) COMMAND ${Python3_EXECUTABLE} -m pybind11_stubgen --ignore-invalid-identifiers '.*' ${TARGET_NAME} -o ${module_binary_stub_temp_dir} DEPENDS ${PYTHON_ACTIVE_PACKAGE_NAME}-modules $ COMMENT "Building stub for python module ${TARGET_NAME}..." - WORKING_DIRECTORY ${module_binary_stub_dir} + WORKING_DIRECTORY ${module_root_binary_dir} ) add_custom_command( From c946406d63e06ffecedc21fa027452ee6525cd59 Mon Sep 17 00:00:00 2001 From: David Gardner Date: Tue, 10 Dec 2024 16:15:50 -0800 Subject: [PATCH 5/6] Don't build stubs for cython modules. We currently don't have any cython modules which we call from Python, and newer versions of stubgen cause unique stubs to be built with each build even in the absense of changes ref: https://github.com/sizmailov/pybind11-stubgen/issues/235 --- cmake/morpheus_utils/python/python_module_tools.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/morpheus_utils/python/python_module_tools.cmake b/cmake/morpheus_utils/python/python_module_tools.cmake index 0ad2b81..ce0bdd0 100644 --- a/cmake/morpheus_utils/python/python_module_tools.cmake +++ b/cmake/morpheus_utils/python/python_module_tools.cmake @@ -613,7 +613,7 @@ macro(__create_python_library MODULE_NAME) # succeed add_dependencies(${PYTHON_ACTIVE_PACKAGE_NAME}-modules ${TARGET_NAME}) - if(_ARGS_BUILD_STUBS) + if(_ARGS_BUILD_STUBS AND NOT _ARGS_IS_CYTHON) # Get the relative path from the project source to the module root cmake_path(RELATIVE_PATH _ARGS_MODULE_ROOT BASE_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE module_root_relative) From 916abf105cb855a84361464311a5c55659e29fef Mon Sep 17 00:00:00 2001 From: David Gardner Date: Wed, 11 Dec 2024 11:27:27 -0800 Subject: [PATCH 6/6] Add --print-safe-value-reprs='.*' to stubgen, meaning to include custom value reprs instead of the ... values stubgen will use --- cmake/morpheus_utils/python/python_module_tools.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/morpheus_utils/python/python_module_tools.cmake b/cmake/morpheus_utils/python/python_module_tools.cmake index ce0bdd0..cd570cc 100644 --- a/cmake/morpheus_utils/python/python_module_tools.cmake +++ b/cmake/morpheus_utils/python/python_module_tools.cmake @@ -625,7 +625,8 @@ macro(__create_python_library MODULE_NAME) # Before installing, create the custom command to generate the stubs add_custom_command( OUTPUT ${module_binary_stub_temp_file} - COMMAND ${Python3_EXECUTABLE} -m pybind11_stubgen --ignore-invalid-identifiers '.*' ${TARGET_NAME} -o ${module_binary_stub_temp_dir} + COMMAND + ${Python3_EXECUTABLE} -m pybind11_stubgen --print-safe-value-reprs='.*' --ignore-invalid-identifiers '.*' ${TARGET_NAME} -o ${module_binary_stub_temp_dir} DEPENDS ${PYTHON_ACTIVE_PACKAGE_NAME}-modules $ COMMENT "Building stub for python module ${TARGET_NAME}..." WORKING_DIRECTORY ${module_root_binary_dir}