Merge branch 'use-requirements-in-ci' into doc-ci-fix

isl-org · Dec 19, 2024 · fe2357d · fe2357d
2 parents 998f1a9 + a07c65c
commit fe2357d
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 37 deletions.
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -27,7 +27,7 @@ jobs:
           python-version: '3.8'
       - name: Install dependencies
         run: |
-          pip install -U clang-format~=10.0.0 yapf==0.30.0 nbformat
+          pip install -U -r python/requirements_style.txt
       - name: Run style check
         run: |
           python util/check_style.py
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -20,11 +20,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  PIP_VER: "23.2.1"
-  WHEEL_VER: "0.38.4"
-  STOOLS_VER: "67.3.2"
-  JEDI_VER: "0.17.2"  # https://github.com/ipython/ipython/issues/12740
-  IDNA_VER: "2.8"  # https://github.com/psf/requests/issues/5710
+  PIP_VER: "24.3.1"
   CUDA_VERSION: "12.1.0"
   SRC_DIR: "D:\\a\\open3d\\open3d"
   BUILD_DIR: "C:\\Open3D\\build"
@@ -213,15 +209,16 @@ jobs:
           cmake --build . --config ${{ matrix.CONFIG }}
           .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test
           Remove-Item "C:\Program Files\Open3D" -Recurse
+      - name: Install Open3D python build requirements
+        working-directory: ${{ env.SOURCE_DIR }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          python -m pip install -U pip==${{ env.PIP_VER }}
+          python -m pip install -U -c python/requirements_build.txt wheel setuptools
       - name: Install Python package
         working-directory: ${{ env.BUILD_DIR }}
         run: |
           $ErrorActionPreference = 'Stop'
-          python -m pip install --upgrade pip==${{ env.PIP_VER }} `
-            wheel==${{ env.WHEEL_VER }} `
-            setuptools==${{ env.STOOLS_VER }} `
-            jedi==${{ env.JEDI_VER }} `
-            idna==${{ env.IDNA_VER }}
           cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package
       - name: Import python package
         # If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python.
@@ -296,13 +293,17 @@ jobs:
             -DBUILD_JUPYTER_EXTENSION=ON `
             ${{ env.SRC_DIR }}
 
+      - name: Install Open3D python build requirements
+        working-directory: ${{ env.SOURCE_DIR }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          python -m pip install -U pip==${{ env.PIP_VER }}
+          python -m pip install -U -c python/requirements_build.txt wheel setuptools
+
       - name: Build Python package
         working-directory: ${{ env.BUILD_DIR }}
         run: |
           $ErrorActionPreference = 'Stop'
-          python -m pip install --upgrade pip==${{ env.PIP_VER }} `
-                                          wheel==${{ env.WHEEL_VER }} `
-                                          setuptools==${{ env.STOOLS_VER }}
           cmake --build . --parallel ${{ env.NPROC }} --config Release --target pip-package
           $PIP_PKG_NAME=(Get-ChildItem lib/python_package/pip_package/open3d*.whl).Name
           echo "PIP_PKG_NAME=$PIP_PKG_NAME"  | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
@@ -365,10 +366,8 @@ jobs:
           python -V
           python -m venv open3d_test_venv
           open3d_test_venv\Scripts\Activate.ps1
-
-          python -m pip install --upgrade pip==${{ env.PIP_VER }} `
-            wheel==${{ env.WHEEL_VER }} `
-            setuptools==${{ env.STOOLS_VER }}
+          python -m pip install -U pip==${{ env.PIP_VER }}
+          python -m pip install -U -c python/requirements_build.txt wheel setuptools
           python -m pip install -U -r python/requirements_test.txt
           $py_tag=(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')")
           if (Test-Path -Path "pip_package") {

diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci
@@ -163,11 +163,10 @@ RUN ./util/install_deps_ubuntu.sh assume-yes
 # Open3D Python dependencies
 RUN source util/ci_utils.sh \
  && if [ "${BUILD_CUDA_MODULE}" = "ON" ]; then \
-        install_python_dependencies with-cuda with-jupyter; \
+        install_python_dependencies with-cuda with-jupyter with-unit-test; \
     else \
-        install_python_dependencies with-jupyter; \
+        install_python_dependencies with-jupyter with-unit-test; \
     fi \
- && pip install -r python/requirements_test.txt
 
 # Open3D Jupyter dependencies
 RUN mkdir -p /etc/apt/keyrings \

diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel
@@ -111,6 +111,7 @@ COPY ./util/ci_utils.sh /root/Open3D/util/
 COPY ./python/requirements.txt /root/Open3D/python/
 COPY ./python/requirements_jupyter_build.txt /root/Open3D/python/
 COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/
+COPY ./python/requirements_build.txt /root/Open3D/python/
 RUN source /root/Open3D/util/ci_utils.sh \
  && install_python_dependencies with-jupyter
 

diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -1,3 +1,10 @@
 [build-system]
-requires = ["ipywidgets>=8.0.3", "pygments>=2.7.4", "jupyter_packaging~=0.12", "jupyterlab>=3.0.0,==3.*", "setuptools>=50.3.2", "wheel==0.38.4"]
+requires = [
+    "ipywidgets>=8.0.4",
+    "pygments>=2.7.4",
+    "jupyter_packaging~=0.12",
+    "jupyterlab>=3.0.0,==3.*",
+    "setuptools>=67.3.2",
+    "wheel==0.38.4",
+]
 build-backend = "setuptools.build_meta"
diff --git a/util/ci_utils.sh b/util/ci_utils.sh
@@ -29,10 +29,7 @@ TENSORFLOW_VER="2.16.2"
 TORCH_VER="2.2.2"
 TORCH_REPO_URL="https://download.pytorch.org/whl/torch/"
 # Python
-PIP_VER="23.2.1"
-WHEEL_VER="0.38.4"
-STOOLS_VER="67.3.2"
-YAPF_VER="0.30.0"
+PIP_VER="24.3.1"
 PROTOBUF_VER="4.24.0"
 
 OPEN3D_INSTALL_DIR=~/open3d_install
@@ -42,10 +39,10 @@ install_python_dependencies() {
 
     echo "Installing Python dependencies"
     options="$(echo "$@" | tr ' ' '|')"
-    python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \
-        setuptools=="$STOOLS_VER"
+    python -m pip install -U pip=="$PIP_VER"
+    python -m pip install -U -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel setuptools
     if [[ "with-unit-test" =~ ^($options)$ ]]; then
-        python -m pip install -U -r python/requirements_test.txt
+        python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt"
     fi
     if [[ "with-cuda" =~ ^($options)$ ]]; then
         TF_ARCH_NAME=tensorflow
@@ -64,7 +61,6 @@ install_python_dependencies() {
         TORCH_GLNX="torch==${TORCH_VER}+cpu"
     fi
 
-    # TODO: modify other locations to use requirements.txt
     python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements.txt"
     if [[ "with-jupyter" =~ ^($options)$ ]]; then
         python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_jupyter_build.txt"
@@ -88,7 +84,7 @@ install_python_dependencies() {
         fi
     fi
     if [ "$BUILD_TENSORFLOW_OPS" == "ON" ] || [ "$BUILD_PYTORCH_OPS" == "ON" ]; then
-        python -m pip install -U yapf=="$YAPF_VER"
+        python -m pip install -U -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" yapf
         # Fix Protobuf compatibility issue
         # https://stackoverflow.com/a/72493690/1255535
         # https://github.com/protocolbuffers/protobuf/issues/10051
@@ -241,8 +237,8 @@ test_wheel() {
     python -m venv open3d_test.venv
     # shellcheck disable=SC1091
     source open3d_test.venv/bin/activate
-    python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \
-        setuptools=="$STOOLS_VER"
+    python -m pip install -U pip=="$PIP_VER"
+    python -m pip install -U -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel setuptools
     echo -n "Using python: $(command -v python)"
     python --version
     echo -n "Using pip: "
@@ -285,7 +281,7 @@ test_wheel() {
 run_python_tests() {
     # shellcheck disable=SC1091
     source open3d_test.venv/bin/activate
-    python -m pip install -U -r python/requirements_test.txt
+    python -m pip install -U -r "$OPEN3D_SOURCE_ROOT/python/requirements_test.txt"
     echo Add --randomly-seed=SEED to the test command to reproduce test order.
     pytest_args=("$OPEN3D_SOURCE_ROOT"/python/test/)
     if [ "$BUILD_PYTORCH_OPS" == "OFF" ] && [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then
@@ -352,9 +348,8 @@ install_docs_dependencies() {
     echo Install Python dependencies for building docs
     command -v python
     python -V
-    python -m pip install -U -q "wheel==$WHEEL_VER" \
-        "pip==$PIP_VER"
-    python -m pip install -U -q "yapf==$YAPF_VER"
+    python -m pip install -U -q "pip==$PIP_VER"
+    python -m pip install -U -q -c "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" wheel yapf
     if [[ -d "$1" ]]; then
         OPEN3D_ML_ROOT="$1"
         echo Installing Open3D-ML dependencies from "${OPEN3D_ML_ROOT}"