From 1c84344a949494b6b750f252435fd4c4a39fc5fd Mon Sep 17 00:00:00 2001 From: Sameer Sheorey <41028320+ssheorey@users.noreply.github.com> Date: Mon, 14 Nov 2022 18:03:36 -0800 Subject: [PATCH] Build with libomp 11.1 and package it in the macOS wheel (#5679) Fail build if newer libomp found Install libomp 11.1.0 from old brew bottle for catalina (10.15) --- .github/workflows/macos.yml | 15 +++++++++++++-- cpp/pybind/CMakeLists.txt | 27 ++++++++++++++++++++++++--- docs/getting_started.in.rst | 10 ---------- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 1b63352d5a7..2f92ca3ab84 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -20,7 +20,7 @@ concurrency: env: # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources - NPROC: 3 + NPROC: 4 DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} jobs: @@ -59,7 +59,13 @@ jobs: - name: Install dependencies run: | brew install ccache pkg-config - brew link --force libomp # needed for cmake to find OpenMP + # Install libomp 11.1.0 from old brew bottle for catalina (10.15). + # Directly installing the Ruby formula will install for the current OS + # https://github.com/microsoft/LightGBM/issues/4229 + brew unlink libomp + curl -L -H "Authorization: Bearer QQ==" -o libomp-11.1.0.catalina.bottle.tar.gz \ + https://ghcr.io/v2/homebrew/core/libomp/blobs/sha256:45a5aa653bd45bd5ff5858580b1a4670c4b5a51ea29d68d45a53f72f56010e05 + brew install -f libomp-11.1.0.catalina.bottle.tar.gz ccache -M 2G # See .github/workflows/readme.md for ccache strategy. - name: Config and build run: | @@ -170,7 +176,12 @@ jobs: cmake --version source util/ci_utils.sh install_python_dependencies + # Install libomp 11.1.0. See comment above. brew install ccache + brew unlink libomp + curl -L -H "Authorization: Bearer QQ==" -o libomp-11.1.0.catalina.bottle.tar.gz \ + https://ghcr.io/v2/homebrew/core/libomp/blobs/sha256:45a5aa653bd45bd5ff5858580b1a4670c4b5a51ea29d68d45a53f72f56010e05 + brew install -f libomp-11.1.0.catalina.bottle.tar.gz ccache -M 2G # See .github/workflows/readme.md for ccache strategy. - name: Config and build wheel diff --git a/cpp/pybind/CMakeLists.txt b/cpp/pybind/CMakeLists.txt index d4cd6e6cab2..3037d2d4325 100644 --- a/cpp/pybind/CMakeLists.txt +++ b/cpp/pybind/CMakeLists.txt @@ -65,9 +65,30 @@ if (BUILD_GUI AND CMAKE_SYSTEM_NAME STREQUAL "Linux") list(APPEND PYTHON_EXTRA_LIBRARIES ${CPP_LIBRARY}.1 ${CPPABI_LIBRARY} ${MESA_CPU_GL_LIBRARY}) endif() +if (WITH_OPENMP AND APPLE AND NOT BUILD_SHARED_LIBS) +# Package libomp v11.1.0, if it is not installed. Later version cause crash on +# x86_64 if PyTorch is already imported. Case of shared libopen3d.dylib is not +# handled. +# https://github.com/microsoft/LightGBM/issues/4229 + list(APPEND PYTHON_EXTRA_LIBRARIES ${OpenMP_libomp_LIBRARY}) + execute_process(COMMAND brew list --versions libomp + COMMAND cut "-d\ " -f2 + OUTPUT_VARIABLE libomp_ver OUTPUT_STRIP_TRAILING_WHITESPACE) + if(libomp_ver VERSION_GREATER_EQUAL 12.0.0) + message(SEND_ERROR "libomp ${libomp_ver} found, which can lead to interoperability problems with other Python libraries using libomp. Please use libomp v11.1 to build Open3D.") + endif() + file(GENERATE OUTPUT update_pybind_libomp.sh + CONTENT [=[libomp_library=$(dyld_info -dependents "$" | +grep libomp | tr -d '[:space:]') +install_name_tool -change $libomp_library @loader_path/../libomp.dylib \ +"$"]=]) + add_custom_command(TARGET pybind POST_BUILD + COMMAND bash update_pybind_libomp.sh + COMMENT "Updating pybind to use packaged libomp") +endif() # Use `make python-package` to create the python package in the build directory -# The python package will be created at PYTHON_PACKAGE_DIR. It contains: +# The python package will be created at PYTHON_PACKAGE_DST_DIR. It contains: # 1) Pure-python code and misc files, copied from python/package # 2) The compiled python-C++ module, i.e. open3d.so (or the equivalents) # 3) Configured files and supporting files @@ -84,8 +105,8 @@ set(GENERATED_OUTPUTS "") # add additional optional compiled modules if (BUILD_TENSORFLOW_OPS) - list( APPEND COMPILED_MODULE_PATH_LIST $ ) - add_custom_command( OUTPUT "${CMAKE_BINARY_DIR}/lib/ml/tf/python/ops/ops.py" + list(APPEND COMPILED_MODULE_PATH_LIST $ ) + add_custom_command(OUTPUT "${CMAKE_BINARY_DIR}/lib/ml/tf/python/ops/ops.py" COMMAND ${Python3_EXECUTABLE} generate_tf_ops_wrapper.py --input "${PYTHON_PACKAGE_SRC_DIR}/open3d/ml/tf/python/ops/ops.py.in" --output "${CMAKE_BINARY_DIR}/lib/ml/tf/python/ops/ops.py" --lib $ DEPENDS open3d_tf_ops WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/docs/getting_started.in.rst b/docs/getting_started.in.rst index 1fb2c7e05aa..e20c8da9735 100644 --- a/docs/getting_started.in.rst +++ b/docs/getting_started.in.rst @@ -43,16 +43,6 @@ Pip (PyPI) ``pip install -U pip>=20.3`` -.. note:: - In MacOS, please install ``libomp`` (OpenMP) on your system before installing - Open3D. - - ``brew install libomp`` - - We do not package OpenMP runtime with Open3D since a system OpenMP runtime - ensures more reliable interoperation with other Python packages (e.g. - PyTorch) that also use OpenMP. - .. note:: In general, we recommend using a `virtual environment `_