diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 653cdee..62c36b4 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -2,10 +2,6 @@ name: CD on: workflow_dispatch: - # pull_request: - # push: - # branches: - # - main release: types: - published @@ -16,33 +12,295 @@ concurrency: env: FORCE_COLOR: 3 + CIBW_BUILD_VERBOSITY: 2 + CIBW_BUILD_FRONTEND: 'pip' + CIBW_SKIP: "pp* *musllinux*" jobs: - dist: - name: Distribution build + sdist: + name: Build source distribution runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install dependencies + run: | + python -m pip install build + + - name: Build source distribution + run: | + python -m build --sdist --outdir dist/ + + - uses: actions/upload-artifact@v4 + with: + name: source_distribution + path: dist + + windows_wheels: + strategy: + fail-fast: false + name: Windows wheels + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.8 + + - name: Set up Go toolchain + uses: actions/setup-go@v5 + with: + go-version: '1.21.x' + cache: false + check-latest: true + + - name: Install MinGW compiler(s) + run: choco install mingw + + - uses: pypa/cibuildwheel@v2.16.2 + with: + package-dir: . + output-dir: wheelhouse + env: + CIBW_ARCHS_WINDOWS: AMD64 + CIBW_TEST_COMMAND: > + hugo version + hugo env --logLevel debug + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: windows_wheels + path: ./wheelhouse/*.whl + if-no-files-found: error + + linux_amd64_wheels: + strategy: + fail-fast: false + name: Linux wheels (amd64) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.8 + + - uses: pypa/cibuildwheel@v2.16.2 + with: + package-dir: . + output-dir: wheelhouse + env: + CIBW_ARCHS_LINUX: x86_64 + CIBW_BEFORE_ALL_LINUX: > + yum install -y wget && + wget https://golang.org/dl/go1.21.5.linux-amd64.tar.gz && + mkdir $HOME/go_installed && + tar -C $HOME/go_installed/ -xzf go1.21.5.linux-amd64.tar.gz && + export PATH="$HOME/go_installed/go/bin:$PATH" && + go version + CIBW_ENVIRONMENT_LINUX: PATH=$PATH:$HOME/go_installed/go/bin + CIBW_REPAIR_WHEEL_COMMAND_LINUX: '' + CIBW_TEST_COMMAND: > + hugo version + hugo env --logLevel debug + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: linux_amd64_wheels + path: ./wheelhouse/*.whl + if-no-files-found: error + + linux_aarch64_wheels: + strategy: + fail-fast: false + name: Linux wheels (aarch64) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.8 + - name: Set up QEMU for emulation + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - uses: pypa/cibuildwheel@v2.16.2 + with: + package-dir: . + output-dir: wheelhouse + env: + CIBW_ARCHS_LINUX: aarch64 + CIBW_BEFORE_ALL_LINUX: bash scripts/ci/tools/linux/install_go.sh + CIBW_ENVIRONMENT_LINUX: PATH=$PATH:$HOME/go_installed/go/bin + CIBW_REPAIR_WHEEL_COMMAND_LINUX: '' + CIBW_TEST_COMMAND: > + hugo version + hugo env --logLevel debug + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: linux_aarch64_wheels + path: ./wheelhouse/*.whl + if-no-files-found: error + + macos_x86_64_wheels: + strategy: + fail-fast: false + name: macOS wheels (x86_64) + runs-on: macos-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: hynek/build-and-inspect-python-package@v2 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.8 + + - name: Set up Go toolchain + uses: actions/setup-go@v5 + with: + go-version: '1.21.x' + cache: false + check-latest: true + + - uses: pypa/cibuildwheel@v2.16.2 + with: + package-dir: . + output-dir: wheelhouse + env: + CIBW_ARCHS_MACOS: x86_64 + CIBW_REPAIR_WHEEL_COMMAND_MACOS: '' + CIBW_TEST_COMMAND: > + hugo version + hugo env --logLevel debug + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: macos_x86_64_wheels + path: ./wheelhouse/*.whl + if-no-files-found: error + + macos_arm64_wheels: + strategy: + fail-fast: false + name: macOS wheels (arm64) + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.8 + + - name: Set up Go toolchain + uses: actions/setup-go@v5 + with: + go-version: '1.21.x' + cache: false + check-latest: true + + - uses: pypa/cibuildwheel@v2.16.2 + with: + package-dir: . + output-dir: wheelhouse + env: + CIBW_ARCHS_MACOS: arm64 + # These are needed to build arm64 binaries on x86_64 macOS + CIBW_ENVIRONMENT_MACOS: > + GOARCH="arm64" + CIBW_REPAIR_WHEEL_COMMAND_MACOS: '' + CIBW_TEST_COMMAND: > + hugo version + hugo env --logLevel debug + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: macos_arm64_wheels + path: ./wheelhouse/*.whl + if-no-files-found: error + + combined_macos_wheels: + name: Combine macOS wheels + runs-on: macos-latest + needs: [macos_x86_64_wheels, macos_arm64_wheels] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download macOS wheels for both architectures + uses: actions/download-artifact@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Combine macOS wheels + run: | + mkdir wheelhouse_arm64 + mkdir wheelhouse_x86_64 + mkdir wheelhouse_universal2 + cp macos_x86_64_wheels/*.whl wheelhouse_x86_64/ + cp macos_arm64_wheels/*.whl wheelhouse_arm64/ + python -m pip install delocate + python scripts/ci/tools/macos/fuse_wheels.py ./wheelhouse_x86_64 ./wheelhouse_arm64 ./wheelhouse_universal2 + + - name: Upload combined macOS wheels + uses: actions/upload-artifact@v4 + with: + name: combined_macos_wheels + path: ./wheelhouse_universal2/*.whl + if-no-files-found: error publish: - needs: [dist] - name: Publish to PyPI - environment: pypi + needs: [sdist, windows_wheels, linux_amd64_wheels, linux_aarch64_wheels, combined_macos_wheels] + name: Publish to PyPI or TestPyPI + environment: release permissions: id-token: write runs-on: ubuntu-latest if: github.event_name == 'release' && github.event.action == 'published' steps: - - uses: actions/download-artifact@v4 - with: - name: Packages - path: dist + - name: Download all artifacts + uses: actions/download-artifact@v4 + + - name: Move all artifacts to upload directory + run: | + mkdir upload + mv source_distribution/* windows_wheels/* linux_amd64_wheels/* linux_aarch64_wheels/* combined_macos_wheels/* upload/ - uses: pypa/gh-action-pypi-publish@release/v1 if: github.event_name == 'release' && github.event.action == 'published' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4837d2e..5013364 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,13 +16,12 @@ env: jobs: build: - name: Wheels (${{ matrix.runs-on }} / Python ${{ matrix.python-version }} / Go ${{ matrix.go-version }}) + name: Wheels (${{ matrix.runs-on }} / Python ${{ matrix.python-version }}) runs-on: ${{ matrix.runs-on }} strategy: fail-fast: false matrix: python-version: ["3.8", "3.12"] - go-version: ["1.20.x", "1.21.x"] runs-on: [ubuntu-latest, macos-latest, windows-latest] steps: @@ -38,7 +37,7 @@ jobs: - name: Set up Go toolchain uses: actions/setup-go@v5 with: - go-version: ${{ matrix.go-version }} + go-version: "1.21.x" cache: false check-latest: true @@ -54,20 +53,20 @@ jobs: uses: actions/cache/restore@v3 with: path: ./hugo_cache/ - key: ${{ runner.os }}-${{ matrix.go-version }}-hugo-build-cache-${{ hashFiles('**/setup.py', '**/pyproject.toml') }} + key: ${{ runner.os }}-hugo-build-cache-${{ hashFiles('**/setup.py', '**/pyproject.toml') }} - name: Install Python dependencies run: python -m pip install build virtualenv nox - name: Build binary distribution (wheel) run: | - python -m build --wheel . --outdir dist/ + python -m build --wheel . --outdir wheelhouse/ - name: Save Hugo builder cache uses: actions/cache/save@v3 with: path: ./hugo_cache/ - key: ${{ runner.os }}-${{ matrix.go-version }}-hugo-build-cache-${{ hashFiles('**/setup.py', '**/pyproject.toml') }} + key: ${{ runner.os }}-hugo-build-cache-${{ hashFiles('**/setup.py', '**/pyproject.toml') }} - name: Test entry points for package run: nox -s venv diff --git a/MANIFEST.in b/MANIFEST.in index 5164aea..5cb5869 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include licenses/LICENSE-hugo.txt -exclude python_hugo/binaries/* +include python_hugo/binaries/* +exclude python_hugo/binaries/hugo-* diff --git a/README.md b/README.md index 3aa713c..45857d1 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,13 @@ -Binaries for the Hugo static site generator, installable via `pip` +Binaries for the extended version of the Hugo static site generator, installable via `pip` This package provides wheels for [Hugo](https://gohugo.io/) to be used with `pip` on macOS, Linux, and Windows. ## Quickstart -Create a virtual environment and install the package: +Create a virtual environment and install the package (or install it globally on your system): ```bash python -m virtualenv venv # (or your preferred method of creating virtual environments) @@ -55,17 +55,30 @@ For more information on using Hugo and its command-line interface, please refer ## Supported platforms -| Platform | Architecture | Supported | -| -------- | ------------ | ---------------- | -| macOS | x86_64 | ✅ | -| macOS | arm64 | ✅ | -| Linux | amd64 | ✅ | -| Linux | arm64 | Coming soon | -| Windows | x86_64 | ✅ | +A subset of the platforms supported by Hugo itself are supported by `python-hugo`. The plan is to support as many platforms as possible with Python wheels and platform tags. Please refer to the following table for a list of supported platforms and architectures: + +| Platform | Architecture | Supported | +| -------- | --------------- | ------------------------------- | +| macOS | x86_64 (Intel) | ✅ | +| macOS | arm64 (Silicon) | ✅ | +| Linux | amd64 | ✅ | +| Linux | arm64 | ✅ | +| Windows | x86_64 | ✅ | +| Windows | i686 | ❌ Will not receive support[^1] | +| Windows | arm64 | 💡 Probable[^3] | +| DragonFlyBSD | amd64 | ❌ Will not receive support[^2] | +| FreeBSD | amd64 | ❌ Will not receive support[^2] | +| OpenBSD | amd64 | ❌ Will not receive support[^2] | +| NetBSD | amd64 | ❌ Will not receive support[^2] | +| Solaris | amd64 | ❌ Will not receive support[^2] | + +[^1]: Windows 32-bit support is possible to include, but hasn't been included due to the diminishing popularity of i686 instruction set-based systems and the lack of resources to test and build for it. If you need support for Windows 32-bit, please consider either using the official Hugo binaries or compiling from [HugoReleaser](https://github.com/gohugoio/hugoreleaser). +[^2]: Support for these platforms is not possible to include because of i. the lack of resources to test and build for them and ii. the lack of support for these platform specifications in Python packaging standards and tooling. If you need support for these platforms, please consider downloading the [official Hugo binaries](https://github.com/gohugoio/hugo/releases) +[^3]: Support for Windows ARM64 is possible to include, but `cibuildwheel` support for it is currently experimental. Hugo does not officially support Windows ARM64 at the moment, but it should be possible to build from source for it and can receive support in the future through this package. ## Building from source -Building Hugo from source requires the following dependencies: +Building the extended version of Hugo from source requires the following dependencies: - [Go](https://go.dev/doc/install) toolchain - [Git](https://git-scm.com/downloads) diff --git a/noxfile.py b/noxfile.py index 2a34f87..e805285 100644 --- a/noxfile.py +++ b/noxfile.py @@ -80,8 +80,13 @@ def release(session: nox.Session) -> None: @nox.session(name="venv") def venv(session: nox.Session) -> None: - """Create a virtual environment and install wheels from the dist/ folder into it.""" - for file in DIR.joinpath("dist").glob("*.whl"): + """Create a virtual environment and install wheels from a specified folder into it.""" + if session.interactive: + folder = "dist" + else: + folder = "wheelhouse" + session.log(f"Installing wheels from {folder}") + for file in DIR.joinpath(folder).glob("*.whl"): session.install(file) - session.run("hugo", "version") - session.run("hugo", "env", "--logLevel", "debug") + session.run("hugo", "version") + session.run("hugo", "env", "--logLevel", "debug") diff --git a/python_hugo/cli.py b/python_hugo/cli.py index b61ac9e..b65fcfe 100644 --- a/python_hugo/cli.py +++ b/python_hugo/cli.py @@ -26,7 +26,6 @@ "aarch64": "arm64", }[platform.machine()] - @lru_cache(maxsize=1) def hugo_executable(): """ @@ -38,9 +37,12 @@ def hugo_executable(): f"hugo-{HUGO_VERSION}-{HUGO_PLATFORM}-{HUGO_ARCH}" + FILE_EXT, ) +MESSAGE = f"Running Hugo {HUGO_VERSION} via python-hugo at {hugo_executable()}" def __call(): """ Hugo binary entry point. Passes all command-line arguments to Hugo. """ + # send to stdout a message that we are using the python-hugo wrapper + print(MESSAGE) os.execvp(hugo_executable(), ["hugo", *sys.argv[1:]]) diff --git a/scripts/ci/tools/linux/install_go.sh b/scripts/ci/tools/linux/install_go.sh new file mode 100644 index 0000000..6513900 --- /dev/null +++ b/scripts/ci/tools/linux/install_go.sh @@ -0,0 +1,23 @@ +# !#/bin/bash + +# Small script to install Golang into a PyPA manylinux2014 Docker container + +yum install -y wget + +arch=$(uname -m) + +if [ "$arch" == "x86_64" ]; then + tarball="go1.21.5.linux-amd64.tar.gz" +elif [[ "$arch" == "aarch64" || "$arch" == "arm64" ]]; then + tarball="go1.21.5.linux-arm64.tar.gz" +else + echo "Unsupported architecture: $arch" + exit 1 +fi + +wget https://golang.org/dl/$tarball +mkdir $HOME/go_installed/ +tar -C $HOME/go_installed/ -xzf $tarball +export PATH=$PATH:$HOME/go_installed/go/bin >> ~/.bashrc +export PATH=$PATH:$HOME/go_installed/go/bin >> ~/.bash_profile +go version diff --git a/scripts/ci/tools/macos/fuse_wheels.py b/scripts/ci/tools/macos/fuse_wheels.py new file mode 100644 index 0000000..9e99d6d --- /dev/null +++ b/scripts/ci/tools/macos/fuse_wheels.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# A wrapper script over delocate-fuse for creating universal2 wheels +# from x86_64 and arm64 wheels, with basic command-line argument parsing + +# Usage: fuse_wheels.py +# Note: these directories must be created beforehand before running this script. + +import sys +import os +import shutil +import subprocess + +in_dir1 = sys.argv[1] +in_dir2 = sys.argv[2] +out_dir = sys.argv[3] + +def search_wheel_in_dir(package, dir): + for i in os.listdir(dir): + if i.startswith(package): + return i + +def copy_if_universal(wheel_name, in_dir, out_dir): + if wheel_name.endswith('universal2.whl') or wheel_name.endswith('any.whl'): + src_path = os.path.join(in_dir, wheel_name) + dst_path = os.path.join( + out_dir, + wheel_name + .replace('x86_64', 'universal2') + .replace('arm64', 'universal2') + ) + + shutil.copy(src_path, dst_path) + return True + else: + return False + +for wheel_name_1 in os.listdir(in_dir1): + package = wheel_name_1.split('-')[0] + wheel_name_2 = search_wheel_in_dir(package, in_dir2) + if copy_if_universal(wheel_name_1, in_dir1, out_dir): + continue + if copy_if_universal(wheel_name_2, in_dir2, out_dir): + continue + + wheel_path_1 = os.path.join(in_dir1, wheel_name_1) + wheel_path_2 = os.path.join(in_dir2, wheel_name_2) + subprocess.run(['delocate-fuse', wheel_path_1, wheel_path_2, '-w', out_dir]) + +for wheel_name in os.listdir(out_dir): + wheel_name_new = wheel_name.replace('x86_64', 'universal2').replace('arm64', 'universal2') + + src_path = os.path.join(out_dir, wheel_name) + dst_path = os.path.join(out_dir, wheel_name_new) + + os.rename(src_path, dst_path) \ No newline at end of file diff --git a/setup.py b/setup.py index 55e9a8e..7ac5b4a 100644 --- a/setup.py +++ b/setup.py @@ -204,9 +204,7 @@ def run(self): for path_spec in files_to_clean: # Make paths absolute and relative to this path - abs_paths = glob.glob( - os.path.normpath(os.path.join(here, path_spec)) - ) + abs_paths = glob.glob(os.path.normpath(os.path.join(here, path_spec))) for path in [str(p) for p in abs_paths]: if not path.startswith(here): # raise error if path in files_to_clean is absolute + outside @@ -243,9 +241,17 @@ def finalize_options(self): if sys.platform == "darwin": platform_tag = get_platform("_") # ensure correct platform tag for macOS arm64 and macOS x86_64 - if "arm64" in platform_tag and os.environ.get("GOARCH") == "amd64": + # macOS 3.12 Python runners are mislabelling the platform tag to be universal2 + # see: https://github.com/pypa/wheel/issues/573. we will explicitly rename the + # universal2 tag to macosx_X_Y_arm64 or macosx_X_Y_x86_64 respectively, since + # we fuse the wheels together later anyway. + if (("arm64" in platform_tag) or ("univeral2" in platform_tag)) and ( + os.environ.get("GOARCH") == "amd64" + ): self.plat_name = platform_tag.replace("arm64", "x86_64") - if "x86_64" in platform_tag and os.environ.get("GOARCH") == "arm64": + if (("x86_64" in platform_tag) or ("universal2" in platform_tag)) and ( + os.environ.get("GOARCH") == "arm64" + ): self.plat_name = platform_tag.replace("x86_64", "arm64") super().finalize_options()