* Introduce macosx-arm64 builds for PyTorch (pull #1463)

bytedeco · Jan 25, 2024 · 96f307e · 96f307e
1 parent bfdd494
commit 96f307e
Show file tree

Hide file tree

Showing 7 changed files with 33 additions and 21 deletions.
diff --git a/.github/workflows/pytorch.yml b/.github/workflows/pytorch.yml
@@ -32,6 +32,10 @@ jobs:
           path: /home/runner/.ccache
       - uses: bytedeco/javacpp-presets/.github/actions/deploy-ubuntu@actions
         timeout-minutes: 350
+  macosx-arm64:
+    runs-on: macos-11
+    steps:
+      - uses: bytedeco/javacpp-presets/.github/actions/deploy-macosx@actions
   macosx-x86_64:
     runs-on: macos-11
 #    strategy:
@@ -61,7 +65,7 @@ jobs:
       - uses: bytedeco/javacpp-presets/.github/actions/deploy-windows@actions
         timeout-minutes: 350
   redeploy:
-    needs: [linux-x86_64, macosx-x86_64, windows-x86_64]
+    needs: [linux-x86_64, macosx-arm64, macosx-x86_64, windows-x86_64]
     runs-on: ubuntu-20.04
     steps:
       - uses: bytedeco/javacpp-presets/.github/actions/redeploy@actions
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,5 @@
 
+ * Introduce `macosx-arm64` builds for PyTorch ([pull #1463](https://github.com/bytedeco/javacpp-presets/pull/1463))
  * Reenable `linux-arm64` builds for CPython and NumPy ([pull #1386](https://github.com/bytedeco/javacpp-presets/pull/1386)
  * Enable Vulkan GPU acceleration for FFmpeg ([pull #1460](https://github.com/bytedeco/javacpp-presets/pull/1460))
  * Include `timeapi.h` for system API of Windows ([pull #1447](https://github.com/bytedeco/javacpp-presets/pull/1447))

diff --git a/pom.xml b/pom.xml
@@ -1467,6 +1467,7 @@
         <module>libraw</module>
         <module>leptonica</module>
         <module>tesseract</module>
+        <module>pytorch</module>
         <module>sentencepiece</module>
       </modules>
       <properties>

diff --git a/pytorch/cppbuild.sh b/pytorch/cppbuild.sh
@@ -56,6 +56,7 @@ git submodule foreach --recursive 'git reset --hard'
 # https://github.com/pytorch/pytorch/pull/66219
 #patch -Np1 < ../../../pytorch.patch
 
+CPYTHON_HOST_PATH="$INSTALL_PATH/../../../cpython/cppbuild/$PLATFORM/host/"
 CPYTHON_PATH="$INSTALL_PATH/../../../cpython/cppbuild/$PLATFORM/"
 OPENBLAS_PATH="$INSTALL_PATH/../../../openblas/cppbuild/$PLATFORM/"
 NUMPY_PATH="$INSTALL_PATH/../../../numpy/cppbuild/$PLATFORM/"
@@ -65,7 +66,12 @@ if [[ -n "${BUILD_PATH:-}" ]]; then
     IFS="$BUILD_PATH_SEPARATOR"
     for P in $BUILD_PATH; do
         if [[ $(find "$P" -name Python.h) ]]; then
-            CPYTHON_PATH="$P"
+            if [[ "$(basename $P)" == "$PLATFORM_HOST" ]]; then
+                CPYTHON_HOST_PATH="$P"
+            fi
+            if [[ "$(basename $P)" == "$PLATFORM" ]]; then
+                CPYTHON_PATH="$P"
+            fi
         elif [[ -f "$P/include/openblas_config.h" ]]; then
             OPENBLAS_PATH="$P"
         elif [[ -f "$P/python/numpy/core/include/numpy/numpyconfig.h" ]]; then
@@ -75,10 +81,12 @@ if [[ -n "${BUILD_PATH:-}" ]]; then
     IFS="$PREVIFS"
 fi
 
+CPYTHON_HOST_PATH="${CPYTHON_HOST_PATH//\\//}"
 CPYTHON_PATH="${CPYTHON_PATH//\\//}"
 OPENBLAS_PATH="${OPENBLAS_PATH//\\//}"
 NUMPY_PATH="${NUMPY_PATH//\\//}"
 
+CPYTHON_PATH="$CPYTHON_HOST_PATH"
 if [[ -f "$CPYTHON_PATH/include/python3.12/Python.h" ]]; then
     # setup.py won't pick up the right libgfortran.so without this
     export LD_LIBRARY_PATH="$OPENBLAS_PATH/lib/:$CPYTHON_PATH/lib/:$NUMPY_PATH/lib/"
@@ -115,7 +123,15 @@ case $PLATFORM in
         export CC="gcc -m64"
         export CXX="g++ -m64"
         ;;
-    macosx-*)
+    macosx-arm64)
+        export CC="clang"
+        export CXX="clang++"
+        export CMAKE_OSX_ARCHITECTURES=arm64 # enable cross-compilation on a x86_64 host machine
+        export USE_MKLDNN=OFF
+        export USE_QNNPACK=OFF # not compatible with arm64 as of PyTorch 2.1.2
+        export CMAKE_OSX_DEPLOYMENT_TARGET=11.00 # minimum needed for arm64 support
+        ;;
+    macosx-x86_64)
         export CC="clang"
         export CXX="clang++"
         ;;

diff --git a/pytorch/platform/gpu/pom.xml b/pytorch/platform/gpu/pom.xml
@@ -31,12 +31,6 @@
       <artifactId>openblas-platform</artifactId>
       <version>0.3.26-${project.parent.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.bytedeco</groupId>
-      <artifactId>numpy-platform</artifactId>
-      <version>1.26.3-${project.parent.version}</version>
-      <optional>true</optional>
-    </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>${javacpp.moduleId}</artifactId>

diff --git a/pytorch/platform/pom.xml b/pytorch/platform/pom.xml
@@ -30,12 +30,6 @@
       <artifactId>openblas-platform</artifactId>
       <version>0.3.26-${project.parent.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.bytedeco</groupId>
-      <artifactId>numpy-platform</artifactId>
-      <version>1.26.3-${project.parent.version}</version>
-      <optional>true</optional>
-    </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>${javacpp.moduleId}</artifactId>

diff --git a/pytorch/pom.xml b/pytorch/pom.xml
@@ -24,12 +24,6 @@
       <artifactId>openblas</artifactId>
       <version>0.3.26-${project.parent.version}</version>
     </dependency>
-    <dependency>
-      <groupId>org.bytedeco</groupId>
-      <artifactId>numpy</artifactId>
-      <version>1.26.3-${project.parent.version}</version>
-      <optional>true</optional>
-    </dependency>
   </dependencies>
 
   <build>
@@ -54,6 +48,12 @@
             <artifactId>numpy-platform</artifactId>
             <version>1.26.3-${project.parent.version}</version>
           </dependency>
+          <dependency>
+            <groupId>org.bytedeco</groupId>
+            <artifactId>cpython</artifactId>
+            <version>3.12.1-${project.parent.version}</version>
+            <classifier>${os.name}-${os.arch}</classifier>
+          </dependency>
         </dependencies>
         <configuration>
           <classPaths>
@@ -91,6 +91,8 @@
             <buildResource>/${javacpp.platform.library.path}/</buildResource>
             <buildResource>/org/bytedeco/openblas/${javacpp.platform}/</buildResource>
             <buildResource>/org/bytedeco/cpython/${javacpp.platform}/</buildResource>
+            <!-- add host cpython for running setup.py when cross-compiling -->
+            <buildResource>/org/bytedeco/cpython/${os.name}-${os.arch}/</buildResource>
             <buildResource>/org/bytedeco/mkldnn/${javacpp.platform}/</buildResource>
             <buildResource>/org/bytedeco/numpy/${javacpp.platform}/</buildResource>
           </buildResources>