From e9ec758d3ec361cac5ccc121eb9118277bff2669 Mon Sep 17 00:00:00 2001 From: Daniel Falbel Date: Tue, 11 Jan 2022 09:46:30 -0300 Subject: [PATCH 01/10] Conditionally include Python in the C++ builds. Added an option `USE_PYTHON` that allows users to decide to link to Python3 or not. --- CMakeLists.txt | 13 +++++++++++-- cmake/TorchVisionConfig.cmake.in | 6 ++++-- test/tracing/frcnn/CMakeLists.txt | 3 --- torchvision/csrc/io/image/image.cpp | 4 ++++ torchvision/csrc/io/video_reader/video_reader.cpp | 2 ++ torchvision/csrc/vision.cpp | 4 +++- 6 files changed, 24 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5082f374736..87179c1471e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,7 @@ set(CMAKE_CXX_STANDARD 14) file(STRINGS version.txt TORCHVISION_VERSION) option(WITH_CUDA "Enable CUDA support" OFF) +option(USE_PYTHON "Link to Python when building" OFF) if(WITH_CUDA) enable_language(CUDA) @@ -17,7 +18,10 @@ if(WITH_CUDA) endif() endif() -find_package(Python3 COMPONENTS Development) +if (USE_PYTHON) + add_definitions(-DUSE_PYTHON) + find_package(Python3 REQUIRED COMPONENTS Development) +endif() find_package(Torch REQUIRED) find_package(PNG REQUIRED) @@ -76,7 +80,12 @@ FOREACH(DIR ${ALLOW_LISTED}) ENDFOREACH() add_library(${PROJECT_NAME} SHARED ${ALL_SOURCES}) -target_link_libraries(${PROJECT_NAME} PRIVATE ${TORCH_LIBRARIES} ${PNG_LIBRARY} ${JPEG_LIBRARIES} Python3::Python) +target_link_libraries(${PROJECT_NAME} PRIVATE ${TORCH_LIBRARIES} ${PNG_LIBRARY} ${JPEG_LIBRARIES}) + +if (USE_PYTHON) + target_link_libraries(${PROJECT_NAME} PRIVATE Python3::Python) +endif() + set_target_properties(${PROJECT_NAME} PROPERTIES EXPORT_NAME TorchVision INSTALL_RPATH ${TORCH_INSTALL_PREFIX}/lib) diff --git a/cmake/TorchVisionConfig.cmake.in b/cmake/TorchVisionConfig.cmake.in index 42a3d566166..aa042844753 100644 --- a/cmake/TorchVisionConfig.cmake.in +++ b/cmake/TorchVisionConfig.cmake.in @@ -28,8 +28,10 @@ include("${CMAKE_CURRENT_LIST_DIR}/${PN}Targets.cmake") if(NOT TARGET torch_library) find_package(Torch REQUIRED) endif() -if(NOT TARGET Python3::Python) -find_package(Python3 COMPONENTS Development) +if (@USE_PYTHON@) + if(NOT TARGET Python3::Python) + find_package(Python3 COMPONENTS Development) + endif() endif() set_target_properties(TorchVision::TorchVision PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${${PN}_INCLUDE_DIR}" INTERFACE_LINK_LIBRARIES "torch;Python3::Python" ) diff --git a/test/tracing/frcnn/CMakeLists.txt b/test/tracing/frcnn/CMakeLists.txt index c79382470bd..324cd363349 100644 --- a/test/tracing/frcnn/CMakeLists.txt +++ b/test/tracing/frcnn/CMakeLists.txt @@ -4,9 +4,6 @@ project(test_frcnn_tracing) find_package(Torch REQUIRED) find_package(TorchVision REQUIRED) -# This due to some headers importing Python.h -find_package(Python3 COMPONENTS Development) - add_executable(test_frcnn_tracing test_frcnn_tracing.cpp) target_compile_features(test_frcnn_tracing PUBLIC cxx_range_for) target_link_libraries(test_frcnn_tracing ${TORCH_LIBRARIES} TorchVision::TorchVision Python3::Python) diff --git a/torchvision/csrc/io/image/image.cpp b/torchvision/csrc/io/image/image.cpp index 13a73829e86..3c9d632f030 100644 --- a/torchvision/csrc/io/image/image.cpp +++ b/torchvision/csrc/io/image/image.cpp @@ -1,16 +1,20 @@ #include "image.h" #include +#ifdef USE_PYTHON #include +#endif // If we are in a Windows environment, we need to define // initialization functions for the _custom_ops extension +#ifdef USE_PYTHON #ifdef _WIN32 PyMODINIT_FUNC PyInit_image(void) { // No need to do anything. return NULL; } #endif +#endif // USE_PYTHON namespace vision { namespace image { diff --git a/torchvision/csrc/io/video_reader/video_reader.cpp b/torchvision/csrc/io/video_reader/video_reader.cpp index cca89483d42..72ed35f9fb8 100644 --- a/torchvision/csrc/io/video_reader/video_reader.cpp +++ b/torchvision/csrc/io/video_reader/video_reader.cpp @@ -1,6 +1,8 @@ #include "video_reader.h" +#ifdef USE_PYTHON #include +#endif #include "../decoder/memory_buffer.h" #include "../decoder/sync_decoder.h" diff --git a/torchvision/csrc/vision.cpp b/torchvision/csrc/vision.cpp index cc8c95254cb..62e1ac37937 100644 --- a/torchvision/csrc/vision.cpp +++ b/torchvision/csrc/vision.cpp @@ -1,7 +1,9 @@ #include "vision.h" #ifndef MOBILE -#include +#ifdef USE_PYTHON +#include From ff243bcfc7d59c21738fe24369a006e12e5d27be Mon Sep 17 00:00:00 2001 From: Daniel Falbel Date: Tue, 11 Jan 2022 11:06:12 -0300 Subject: [PATCH 02/10] Also conditionally include this defintion. --- torchvision/csrc/io/video_reader/video_reader.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torchvision/csrc/io/video_reader/video_reader.cpp b/torchvision/csrc/io/video_reader/video_reader.cpp index 72ed35f9fb8..ec4857a8399 100644 --- a/torchvision/csrc/io/video_reader/video_reader.cpp +++ b/torchvision/csrc/io/video_reader/video_reader.cpp @@ -7,6 +7,7 @@ #include "../decoder/memory_buffer.h" #include "../decoder/sync_decoder.h" +#ifdef USE_PYTHON // If we are in a Windows environment, we need to define // initialization functions for the _custom_ops extension #ifdef _WIN32 @@ -15,6 +16,7 @@ PyMODINIT_FUNC PyInit_video_reader(void) { return NULL; } #endif +#endif // USE_PYTHONs using namespace ffmpeg; From ba4ba3c15dd5a9b7449c788f1fa8dae966ebb17a Mon Sep 17 00:00:00 2001 From: Daniel Falbel Date: Tue, 11 Jan 2022 11:07:04 -0300 Subject: [PATCH 03/10] Define `USE_PYTHON` for the Windows builds. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 36d95c75bec..cfbfa0d972f 100644 --- a/setup.py +++ b/setup.py @@ -201,7 +201,7 @@ def get_extensions(): if sys.platform == "win32": define_macros += [("torchvision_EXPORTS", None)] - + define_macros += [("USE_PYTHON", None)] extra_compile_args["cxx"].append("/MP") debug_mode = os.getenv("DEBUG", "0") == "1" From 6aaa1e8a26a892f00e26217211c3b765f30d19fa Mon Sep 17 00:00:00 2001 From: Daniel Falbel Date: Tue, 11 Jan 2022 11:18:19 -0300 Subject: [PATCH 04/10] Remove Python3 reference in CMake test as it's no longer required. --- test/tracing/frcnn/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tracing/frcnn/CMakeLists.txt b/test/tracing/frcnn/CMakeLists.txt index 324cd363349..506953227e1 100644 --- a/test/tracing/frcnn/CMakeLists.txt +++ b/test/tracing/frcnn/CMakeLists.txt @@ -6,5 +6,5 @@ find_package(TorchVision REQUIRED) add_executable(test_frcnn_tracing test_frcnn_tracing.cpp) target_compile_features(test_frcnn_tracing PUBLIC cxx_range_for) -target_link_libraries(test_frcnn_tracing ${TORCH_LIBRARIES} TorchVision::TorchVision Python3::Python) +target_link_libraries(test_frcnn_tracing ${TORCH_LIBRARIES} TorchVision::TorchVision) set_property(TARGET test_frcnn_tracing PROPERTY CXX_STANDARD 14) From 945f805296a88cf075470100f65471beddc28fe1 Mon Sep 17 00:00:00 2001 From: Daniel Falbel Date: Tue, 11 Jan 2022 11:19:36 -0300 Subject: [PATCH 05/10] Accidentally removed the closing > from the decl. --- torchvision/csrc/vision.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchvision/csrc/vision.cpp b/torchvision/csrc/vision.cpp index 62e1ac37937..575049954cf 100644 --- a/torchvision/csrc/vision.cpp +++ b/torchvision/csrc/vision.cpp @@ -2,7 +2,7 @@ #ifndef MOBILE #ifdef USE_PYTHON -#include #endif #endif #include From 196e134829ec6fa667b9ac14320f75ba24abd1cf Mon Sep 17 00:00:00 2001 From: Daniel Falbel Date: Tue, 11 Jan 2022 12:58:43 -0300 Subject: [PATCH 06/10] Also add `USE_PYTHON` when building the separate image library on Windows. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index cfbfa0d972f..7948331fbb0 100644 --- a/setup.py +++ b/setup.py @@ -254,6 +254,9 @@ def get_extensions(): image_library = [] image_link_flags = [] + if sys.platform == "win32": + image_macros += [("USE_PYTHON", None)] + # Locating libPNG libpng = distutils.spawn.find_executable("libpng-config") pngfix = distutils.spawn.find_executable("pngfix") From 46db28d26ba3557c1240673f812ab423551823df Mon Sep 17 00:00:00 2001 From: Daniel Falbel Date: Tue, 11 Jan 2022 13:26:19 -0300 Subject: [PATCH 07/10] Also conditionally include this depening on USE_PYTHON. --- torchvision/csrc/vision.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torchvision/csrc/vision.cpp b/torchvision/csrc/vision.cpp index 575049954cf..161b8ecfa2f 100644 --- a/torchvision/csrc/vision.cpp +++ b/torchvision/csrc/vision.cpp @@ -18,10 +18,12 @@ // initialization functions for the _custom_ops extension. // For PyMODINIT_FUNC to work, we need to include Python.h #if !defined(MOBILE) && defined(_WIN32) +#ifdef USE_PYTHON PyMODINIT_FUNC PyInit__C(void) { // No need to do anything. return NULL; } +#endif // USE_PYTHON #endif // !defined(MOBILE) && defined(_WIN32) namespace vision { From 77e2a47f862dea4aad07e783faeddaf96aa9373f Mon Sep 17 00:00:00 2001 From: Daniel Falbel Date: Tue, 11 Jan 2022 13:28:48 -0300 Subject: [PATCH 08/10] Go back to require Python for this example. --- test/tracing/frcnn/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/tracing/frcnn/CMakeLists.txt b/test/tracing/frcnn/CMakeLists.txt index 506953227e1..c79382470bd 100644 --- a/test/tracing/frcnn/CMakeLists.txt +++ b/test/tracing/frcnn/CMakeLists.txt @@ -4,7 +4,10 @@ project(test_frcnn_tracing) find_package(Torch REQUIRED) find_package(TorchVision REQUIRED) +# This due to some headers importing Python.h +find_package(Python3 COMPONENTS Development) + add_executable(test_frcnn_tracing test_frcnn_tracing.cpp) target_compile_features(test_frcnn_tracing PUBLIC cxx_range_for) -target_link_libraries(test_frcnn_tracing ${TORCH_LIBRARIES} TorchVision::TorchVision) +target_link_libraries(test_frcnn_tracing ${TORCH_LIBRARIES} TorchVision::TorchVision Python3::Python) set_property(TARGET test_frcnn_tracing PROPERTY CXX_STANDARD 14) From 89d5bdf9a72d95c1764b730c3dcb29222d4b0a69 Mon Sep 17 00:00:00 2001 From: Daniel Falbel Date: Tue, 11 Jan 2022 13:54:33 -0300 Subject: [PATCH 09/10] Find Python in the hello world example. --- examples/cpp/hello_world/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/cpp/hello_world/CMakeLists.txt b/examples/cpp/hello_world/CMakeLists.txt index 3244efb392b..3ca59e4c199 100644 --- a/examples/cpp/hello_world/CMakeLists.txt +++ b/examples/cpp/hello_world/CMakeLists.txt @@ -6,6 +6,10 @@ project(hello-world) # so there is no need to also add `find_package(Torch)` here. find_package(TorchVision REQUIRED) +# This due to LibTorch's version is the one included in the Python +# package that links to Python. +find_package(Python3 COMPONENTS Development) + add_executable(hello-world main.cpp) # We now need to link the TorchVision library to our executable. From a2594f58c4e9d39b71d3d9eba2be31a06053ccab Mon Sep 17 00:00:00 2001 From: Daniel Falbel Date: Wed, 12 Jan 2022 10:10:45 -0300 Subject: [PATCH 10/10] Add a paragraph documenting the `USE_PYTHON` option. --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index ced9248512a..114e8d709d7 100644 --- a/README.rst +++ b/README.rst @@ -157,6 +157,10 @@ so make sure that it is also available to cmake via the ``CMAKE_PREFIX_PATH``. For an example setup, take a look at ``examples/cpp/hello_world``. +Python linking is disabled by default when compiling TorchVision with CMake, this allows you to run models without any Python +dependency. In some special cases where TorchVision's operators are used from Python code, you may need to link to Python. This +can be done by passing ``-DUSE_PYTHON=on`` to CMake. + TorchVision Operators --------------------- In order to get the torchvision operators registered with torch (eg. for the JIT), all you need to do is to ensure that you