diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 47d5c94d..3631da52 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,31 +2,27 @@ name: CI build on: push: branches: - - master + - master pull_request: branches: - - master - + - master # Global Settings env: PYTHON_VERSION: "3.7" GODOT_BINARY_VERSION: "3.2.3" - jobs: - - static-checks: - name: '📊 Static checks' + name: "📊 Static checks" runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 with: - submodules: true + submodules: true - name: Set up Python - uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 with: python-version: ${{ env.PYTHON_VERSION }} - name: Bootstrap @@ -38,26 +34,24 @@ jobs: run: | pre-commit run --all-files --show-diff-on-failure - -################################################################################# - + ################################################################################# linux-build: - name: '🐧 Linux build' + name: "🐧 Linux build" runs-on: ubuntu-latest env: CC: clang - PLATFORM: 'x11-64' + PLATFORM: "x11-64" steps: - - name: 'Checkout' - uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + - name: "Checkout" + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 with: - submodules: true - - name: 'Set up Python' - uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + submodules: true + - name: "Set up Python" + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 with: python-version: ${{ env.PYTHON_VERSION }} - - name: 'Setup venv' + - name: "Setup venv" run: | set -eux ${{ env.CC }} --version @@ -68,57 +62,55 @@ jobs: echo 'godot_binary = "${{ env.GODOT_BINARY_VERSION }}"' >> custom.py echo 'platform = "${{ env.PLATFORM }}"' >> custom.py echo 'CC = "${{ env.CC }}"' >> custom.py - - name: 'Build project' + - name: "Build project" run: | set -eux scons build -j2 - - name: 'Start xvfb' + - name: "Start xvfb" run: | /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 & echo ">>> Started xvfb" - - name: 'Run tests' + - name: "Run tests" run: | set -eux scons tests headless=true env: - DISPLAY: ':99.0' - - name: 'Generate artifact archive' + DISPLAY: ":99.0" + - name: "Generate artifact archive" run: | set -eux scons release - - name: 'Export release artifact' - uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 + - name: "Export release artifact" + uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 with: name: ${{ env.PLATFORM }}-release - path: 'build/godot-python-*.tar.bz2' - - -################################################################################# + path: "build/godot-python-*.tar.bz2" + ################################################################################# windows-build: - name: '🏁 Windows build' + name: "🏁 Windows build" runs-on: windows-latest strategy: matrix: include: - - PLATFORM: 'windows-64' - PYTHON_ARCH: 'x64' - VS_ARCH: 'amd64' - - PLATFORM: 'windows-32' - PYTHON_ARCH: 'x86' - VS_ARCH: 'x86' + - PLATFORM: "windows-64" + PYTHON_ARCH: "x64" + VS_ARCH: "amd64" + - PLATFORM: "windows-32" + PYTHON_ARCH: "x86" + VS_ARCH: "x86" steps: - - name: 'Checkout' - uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + - name: "Checkout" + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 with: - submodules: true - - name: 'Set up Python' - uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + submodules: true + - name: "Set up Python" + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 with: python-version: ${{ env.PYTHON_VERSION }} architecture: ${{ matrix.PYTHON_ARCH }} - - name: 'Setup venv' + - name: "Setup venv" shell: bash run: | set -eux @@ -131,12 +123,12 @@ jobs: echo 'MSVC_USE_SCRIPT = True' >> custom.py echo 'TARGET_ARCH = "${{ matrix.VS_ARCH }}"' >> custom.py echo 'CC = "cl.exe"' >> custom.py - - name: 'Build project' + - name: "Build project" shell: bash run: | set -eux scons build -j2 - - name: 'Install Mesa3D OpenGL' + - name: "Install Mesa3D OpenGL" shell: bash run: | set -eux @@ -153,41 +145,39 @@ jobs: 7z.exe x mesa.7z ls -lh opengl32.dll # Sanity check popd - - name: 'Run tests' + - name: "Run tests" shell: bash run: | set -eux scons tests - - name: 'Generate artifact archive' + - name: "Generate artifact archive" shell: bash run: | scons release - - name: 'Export release artifact' - uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 + - name: "Export release artifact" + uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 with: name: ${{ matrix.PLATFORM }}-release - path: 'build/godot-python-*.zip' - - -################################################################################# + path: "build/godot-python-*.zip" + ################################################################################# - macos-build: - name: '🍎 macOS build' + macos-intel-build: + name: "🍎 macOS/intel build" runs-on: macos-latest env: CC: clang - PLATFORM: 'osx-64' + PLATFORM: "osx-64" steps: - - name: 'Checkout' - uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + - name: "Checkout" + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 with: - submodules: true - - name: 'Set up Python' - uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + submodules: true + - name: "Set up Python" + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 with: python-version: ${{ env.PYTHON_VERSION }} - - name: 'Setup venv' + - name: "Setup venv" run: | set -eux ${{ env.CC }} --version @@ -201,30 +191,79 @@ jobs: echo 'godot_binary = "${{ env.GODOT_BINARY_VERSION }}"' >> custom.py echo 'platform = "${{ env.PLATFORM }}"' >> custom.py echo 'CC = "${{ env.CC }}"' >> custom.py - - name: 'Build project' + - name: "Build project" run: | set -eux scons build -j2 - - name: 'Run tests' + - name: "Run tests" run: | set -eux scons tests - - name: 'Generate artifact archive' + - name: "Generate artifact archive" run: | set -eux scons release - - name: 'Export release artifact' - uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 + - name: "Export release artifact" + uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 with: name: ${{ env.PLATFORM }}-release - path: 'build/godot-python-*.tar.bz2' + path: "build/godot-python-*.tar.bz2" + ################################################################################# -################################################################################# + macos-m1-build: + name: "🍏 macOS/m1 build" + runs-on: macos-latest + env: + CC: clang + PLATFORM: "osx-arm" + ARCH: "arm64" + steps: + - name: "Checkout" + uses: actions/checkout@f1d3225b5376a0791fdee5a0e8eac5289355e43a # pin@v2 + with: + submodules: true + - name: "Set up Python" + uses: actions/setup-python@0291cefc54fa79cd1986aee8fa5ecb89ad4defea # pin@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: "Setup venv" + run: | + set -eux + ${{ env.CC }} --version + python --version + brew update + brew install zlib openssl + brew install --cask xquartz + pip install -U pip + pip install -r requirements.txt + # Configuration for scons + echo 'godot_binary = "${{ env.GODOT_BINARY_VERSION }}"' >> custom.py + echo 'platform = "${{ env.PLATFORM }}"' >> custom.py + echo 'arch = "${{ env.ARCH }}"' >> custom.py + echo 'CC = "${{ env.CC }}"' >> custom.py + - name: "Build project" + run: | + set -eux + scons build -j2 + # - name: 'Run tests' + # run: | + # set -eux + # scons tests + - name: "Generate artifact archive" + run: | + set -eux + scons release + - name: "Export release artifact" + uses: actions/upload-artifact@11830c9f4d30053679cb8904e3b3ce1b8c00bf40 # pin@v2 + with: + name: ${{ env.PLATFORM }}-{{ env.ARCH }}-release + path: "build/godot-python-*.tar.bz2" + ################################################################################# publish-release: - name: 'Publish ${{ matrix.PLATFORM }} release' + name: "Publish ${{ matrix.PLATFORM }} release" if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest needs: @@ -239,10 +278,10 @@ jobs: - PLATFORM: windows-32 - PLATFORM: osx-64 steps: - - uses: actions/download-artifact@0ede0875b5db9a2824878bbbbe3d758a75eb8268 # pin@v2 + - uses: actions/download-artifact@0ede0875b5db9a2824878bbbbe3d758a75eb8268 # pin@v2 name: ${{ matrix.PLATFORM }}-release - - name: 'Upload release' - uses: svenstaro/upload-release-action@483c1e56f95e88835747b1c7c60581215016cbf2 # pin@v2 + - name: "Upload release" + uses: svenstaro/upload-release-action@483c1e56f95e88835747b1c7c60581215016cbf2 # pin@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} tag: ${{ github.ref }} diff --git a/README.rst b/README.rst index db352867..85aa22b6 100644 --- a/README.rst +++ b/README.rst @@ -244,7 +244,7 @@ For MacOS: godot-python(venv)$ scons platform=osx-64 CC=clang release -Valid platforms are `x11-64`, `x11-32`, `windows-64`, `windows-32` and `osx-64`. +Valid platforms are `x11-64`, `x11-32`, `windows-64`, `windows-32`, `osx-64`, and `osx-arm`. Check Travis or Appveyor links above to see the current status of your platform. This command will checkout CPython repo, move to a pinned commit and build diff --git a/SConstruct b/SConstruct index 69859bd6..411b0009 100644 --- a/SConstruct +++ b/SConstruct @@ -42,9 +42,10 @@ vars.Add( "platform", "Target platform", "", - allowed_values=("x11-64", "x11-32", "windows-64", "windows-32", "osx-64"), + allowed_values=("x11-64", "x11-32", "windows-64", "windows-32", "osx-64", "osx-arm"), ) ) +vars.Add("arch", "Platform-dependent architecture (arm/arm64/x86/x64/mips/...)", allowed_values=("x86_64", "arm64")) vars.Add("pytest_args", "Pytest arguments passed to tests functions", "") vars.Add( "godot_args", "Additional arguments passed to godot binary when running tests&examples", "" diff --git a/misc/release_pythonscript.gdnlib b/misc/release_pythonscript.gdnlib index 1510867c..54db81df 100644 --- a/misc/release_pythonscript.gdnlib +++ b/misc/release_pythonscript.gdnlib @@ -12,6 +12,7 @@ Server.64="res://addons/pythonscript/x11-64/libpythonscript.so" Windows.64="res://addons/pythonscript/windows-64/pythonscript.dll" Windows.32="res://addons/pythonscript/windows-32/pythonscript.dll" OSX.64="res://addons/pythonscript/osx-64/libpythonscript.dylib" +OSX="res://addons/pythonscript/osx/libpythonscript.dylib" [dependencies] diff --git a/platforms/osx-arm/SConscript b/platforms/osx-arm/SConscript new file mode 100644 index 00000000..b4f4c7c7 --- /dev/null +++ b/platforms/osx-arm/SConscript @@ -0,0 +1,155 @@ +import os +import zstandard +import tarfile +import json +import shutil +import subprocess +from pathlib import Path + + +Import("env") + + +cpython_build = Dir("cpython_build") + + +env["bits"] = "64" +env["godot_binary_download_version"] = ('3', '3', '1', 'stable') +env["godot_binary_download_platform"] = "osx.universal" +env["godot_binary_download_zip_path"] = "Godot.app/Contents/MacOS/Godot" +env["cpython_build"] = cpython_build +env["cpython_build_dir"] = cpython_build +env["DIST_SITE_PACKAGES"] = Dir(f"{env['DIST_PLATFORM']}/lib/python3.9/site-packages") + +### Build config for pythonscript ### + +env.AppendUnique(CFLAGS=["-m64"]) +env.AppendUnique(LINKFLAGS=["-m64"]) + +print("Building for macOS 10.15+, platform arm64.") +env.Append(CCFLAGS=["-arch", "arm64", "-mmacosx-version-min=10.15"]) +env.Append(LINKFLAGS=["-arch", "arm64", "-mmacosx-version-min=10.15"]) + +try: + sdk_name = "macosx" + sdk_path = subprocess.check_output(["xcrun", "--sdk", sdk_name, "--show-sdk-path"]).strip().decode("utf-8") + if sdk_path: + env["MACOS_SDK_PATH"] = sdk_path +except (subprocess.CalledProcessError, OSError): + print("Failed to find SDK path while running xcrun --sdk {} --show-sdk-path.".format(sdk_name)) + raise + +env.Append(CPPFLAGS=['-isysroot', '$MACOS_SDK_PATH']) +env.Append(LINKFLAGS=['-isysroot', '$MACOS_SDK_PATH']) + +# Cannot use CPPPATH&LIBPATH here given headers are within `cpython_build` target, +# so Scons consider the headers are a missing target +env.AppendUnique(CFLAGS=[f"-I{cpython_build.abspath}/include/python3.9/"]) +env.AppendUnique(LINKFLAGS=[f"-L{cpython_build.abspath}/lib"]) + + +### Fetch Python prebuild ### +CPYTHON_DOWNLOADS_URL = "https://github.com/indygreg/python-build-standalone/releases/download" +CPYTHON_PREBUILD = f"20210506/cpython-3.9.5-aarch64-apple-darwin-pgo-20210506T0943.tar.zst" +CPYTHON_PREBUILD_URL = f"{CPYTHON_DOWNLOADS_URL}/{CPYTHON_PREBUILD}" + +cpython_prebuild_archive = env.Download( + target=File(CPYTHON_PREBUILD_URL.rsplit("/", 1)[1]), url=CPYTHON_PREBUILD_URL +) +env.NoClean(cpython_prebuild_archive) + + +### Extract prebuild ### + + +def extract_cpython_prebuild(target, source, env): + archive_path = source[0].abspath + target_path = target[0].abspath + with open(archive_path, "rb") as fh: + dctx = zstandard.ZstdDecompressor() + with dctx.stream_reader(fh) as reader: + with tarfile.open(mode="r|", fileobj=reader) as tf: + tf.extractall(target_path) + + +cpython_prebuild_src = env.Command( + Dir("cpython_prebuild"), cpython_prebuild_archive, extract_cpython_prebuild +) +env.NoClean(cpython_prebuild_src) + + +### Generate custom build from the prebuild ### + + +def generate_cpython_build(target, source, env): + build = Path(target[0].abspath) + prebuild = Path(source[0].abspath) / "python" + + # verify correct version + conf = json.loads((prebuild / "PYTHON.json").read_text()) + assert conf["version"] == "7" + assert conf["libpython_link_mode"] == "shared" + + # verify correct triple-target + assert conf["target_triple"] == "aarch64-apple-darwin" + + shutil.copytree(str(prebuild / "install"), str(build), symlinks=True) + shutil.copytree(str(prebuild / "licenses"), str(build / "licenses"), symlinks=True) + + shutil.rmtree(str(build / "share")) + + # Remove static library stuff + config = conf["python_stdlib_platform_config"] + assert config.startswith("install/lib/") + config = build / config[len("install/") :] + assert config.exists() + shutil.rmtree(str(config)) + + # Patch binaries to load libpython3.x.dylib with a relative path + # Lib paths are hardcoded into the executable, and if the lib is not found at the path, then it craps out. + # Unfortunately compiling python will hardcode the absolute path of libpython.dylib into the executable, + # so if you move it around it will break. + # the solution here is to modify the executable and make sure the lib path is not an absolute path, + # but an path relative to @loader_path, which is a special symbol that points to the executable. + # See: http://joaoventura.net/blog/2016/embeddable-python-osx-from-src/ + # and https://stackoverflow.com/questions/7880454/python-executable-not-finding-libpython-shared-library + prebuild_shared_lib_path = conf["build_info"]["core"]["shared_lib"] + path, _ = prebuild_shared_lib_path.rsplit("/", 1) + assert path == "install/lib" # Make sure libpython.so is on lib folder + binary = build / "bin/python3.9" + assert binary.is_file() + dylib = build / "lib/libpython3.9.dylib" + cmd = f"install_name_tool -id @rpath/{dylib.name} {dylib}" + subprocess.run(cmd.split(), check=True) + + stdlib_path = build / "lib/python3.9" + + # Remove tests lib (pretty big and basically useless) + shutil.rmtree(str(stdlib_path / "test")) + + # Also remove __pycache__ & .pyc stuff + for pycache in stdlib_path.glob("**/__pycache__"): + shutil.rmtree(str(pycache)) + + # Make sure site-packages is empty to avoid including pip (ensurepip should be used instead) + shutil.rmtree(str(stdlib_path / "site-packages")) + + # Zip the stdlib to save plenty of space \o/ + if env["compressed_stdlib"]: + tmp_stdlib_path = build / "lib/tmp_python3.9" + shutil.move(str(stdlib_path), str(tmp_stdlib_path)) + stdlib_path.mkdir() + shutil.move(str(tmp_stdlib_path / "lib-dynload"), str(stdlib_path / "lib-dynload")) + shutil.make_archive( + base_name=build / "lib/python39", format="zip", root_dir=str(tmp_stdlib_path) + ) + shutil.rmtree(str(tmp_stdlib_path)) + # Oddly enough, os.py must be present (even if empty !) otherwise + # Python failed to find it home... + (stdlib_path / "os.py").touch() + + (stdlib_path / "site-packages").mkdir() + + +env.Command(cpython_build, cpython_prebuild_src, generate_cpython_build) +env.NoClean(cpython_build) diff --git a/pythonscript/SConscript b/pythonscript/SConscript index 00783e20..de4bc66b 100644 --- a/pythonscript/SConscript +++ b/pythonscript/SConscript @@ -5,7 +5,11 @@ if env["platform"].startswith("windows"): c_env.AppendUnique(LIBS=["python38"]) elif env["platform"].startswith("osx"): - c_env.AppendUnique(LIBS=["python3.8"]) + if env["platform"] == "osx-arm": + c_env.AppendUnique(LIBS=["python3.9"]) + else: + c_env.AppendUnique(LIBS=["python3.8"]) + # if we don't give the lib a proper install_name, macos won't be able to find it, # and will link the cython modules with a relative path c_env.AppendUnique( diff --git a/pythonscript/godot/SConscript b/pythonscript/godot/SConscript index 8db5f212..4245c71b 100644 --- a/pythonscript/godot/SConscript +++ b/pythonscript/godot/SConscript @@ -63,7 +63,8 @@ sample = env["bindings_generate_sample"] # dont strip on debug builds if not sample and not env["debug"]: if env["CC_IS_GCC"]: - bindings_env.AppendUnique(LINKFLAGS=["-Wl,--strip-all"]) + pass + # bindings_env.AppendUnique(LINKFLAGS=["-Wl,--strip-all"]) elif env["CC_IS_CLANG"]: bindings_env.AppendUnique(LINKFLAGS=["-Wl,-s"]) diff --git a/site_scons/site_tools/cython.py b/site_scons/site_tools/cython.py index eeb316b1..4c7bee3c 100644 --- a/site_scons/site_tools/cython.py +++ b/site_scons/site_tools/cython.py @@ -91,6 +91,7 @@ def CythonCompile(env, target, source): f"-Wl,-rpath,'{loader_token}/{libpythonscript_path}'", ] # TODO: use scons `env.LoadableModule` for better macos support ? + version = "python3.9" if env["platform"] == 'osx-arm' else "python3.8" ret = env.SharedLibrary( target=target, source=source, @@ -98,7 +99,7 @@ def CythonCompile(env, target, source): SHLIBSUFFIX=".so", CFLAGS=cflags, LINKFLAGS=[*linkflags, *env["LINKFLAGS"]], - LIBS=["python3.8", "pythonscript"], + LIBS=[version, "pythonscript"], # LIBS=[*env["CYTHON_LIBS"], *env["LIBS"]], # LIBPATH=[*env['CYTHON_LIBPATH'], *env['LIBPATH']] ) diff --git a/tests/bindings/pythonscript.gdnlib b/tests/bindings/pythonscript.gdnlib index 1510867c..7e0c5c68 100644 --- a/tests/bindings/pythonscript.gdnlib +++ b/tests/bindings/pythonscript.gdnlib @@ -12,6 +12,7 @@ Server.64="res://addons/pythonscript/x11-64/libpythonscript.so" Windows.64="res://addons/pythonscript/windows-64/pythonscript.dll" Windows.32="res://addons/pythonscript/windows-32/pythonscript.dll" OSX.64="res://addons/pythonscript/osx-64/libpythonscript.dylib" +OSX.universal="res://addons/pythonscript/osx-arm/libpythonscript.dylib" [dependencies] @@ -21,3 +22,4 @@ Server.64=[] Windows.64=[] Windows.32=[] OSX.64=[] +OSX.universal=[]