🎡 Refactor GitHub Actions workflows, get wheels ready, cross-compile …

…macOS wheels Closes #13 Closes #2 - Cross-compiled macOS wheels and native wheels, fused with helper scripts and `delocate-fuse` later - Linux amd64 and aarch64 wheels (the latter is via QEMU emulation) - Python 3.8–3.12 support - Windows amd64 wheels - Notes on supported architectures and platforms have been added to the README
agriyakhetarpal · Jan 5, 2024 · 4147293 · 4147293
2 parents f3b572e + e980b94
commit 4147293
Show file tree

Hide file tree

Showing 9 changed files with 403 additions and 41 deletions.
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -2,10 +2,6 @@ name: CD
 
 on:
   workflow_dispatch:
-  # pull_request:
-  # push:
-  #   branches:
-  #     - main
   release:
     types:
       - published
@@ -16,33 +12,295 @@ concurrency:
 
 env:
   FORCE_COLOR: 3
+  CIBW_BUILD_VERBOSITY: 2
+  CIBW_BUILD_FRONTEND: 'pip'
+  CIBW_SKIP: "pp* *musllinux*"
 
 jobs:
-  dist:
-    name: Distribution build
+  sdist:
+    name: Build source distribution
     runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Install dependencies
+        run: |
+          python -m pip install build
+
+      - name: Build source distribution
+        run: |
+          python -m build --sdist --outdir dist/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: source_distribution
+          path: dist
+
+  windows_wheels:
+    strategy:
+      fail-fast: false
+    name: Windows wheels
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.8
+
+      - name: Set up Go toolchain
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21.x'
+          cache: false
+          check-latest: true
+
+      - name: Install MinGW compiler(s)
+        run: choco install mingw
+
+      - uses: pypa/cibuildwheel@v2.16.2
+        with:
+          package-dir: .
+          output-dir: wheelhouse
+        env:
+          CIBW_ARCHS_WINDOWS: AMD64
+          CIBW_TEST_COMMAND: >
+            hugo version
+            hugo env --logLevel debug
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: windows_wheels
+          path: ./wheelhouse/*.whl
+          if-no-files-found: error
+
+  linux_amd64_wheels:
+    strategy:
+      fail-fast: false
+    name: Linux wheels (amd64)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.8
+
+      - uses: pypa/cibuildwheel@v2.16.2
+        with:
+          package-dir: .
+          output-dir: wheelhouse
+        env:
+          CIBW_ARCHS_LINUX: x86_64
+          CIBW_BEFORE_ALL_LINUX: >
+            yum install -y wget &&
+            wget https://golang.org/dl/go1.21.5.linux-amd64.tar.gz &&
+            mkdir $HOME/go_installed &&
+            tar -C $HOME/go_installed/ -xzf go1.21.5.linux-amd64.tar.gz &&
+            export PATH="$HOME/go_installed/go/bin:$PATH" &&
+            go version
+          CIBW_ENVIRONMENT_LINUX: PATH=$PATH:$HOME/go_installed/go/bin
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: ''
+          CIBW_TEST_COMMAND: >
+            hugo version
+            hugo env --logLevel debug
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: linux_amd64_wheels
+          path: ./wheelhouse/*.whl
+          if-no-files-found: error
+
+  linux_aarch64_wheels:
+    strategy:
+      fail-fast: false
+    name: Linux wheels (aarch64)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.8
 
+      - name: Set up QEMU for emulation
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: all
+
+      - uses: pypa/cibuildwheel@v2.16.2
+        with:
+          package-dir: .
+          output-dir: wheelhouse
+        env:
+          CIBW_ARCHS_LINUX: aarch64
+          CIBW_BEFORE_ALL_LINUX: bash scripts/ci/tools/linux/install_go.sh
+          CIBW_ENVIRONMENT_LINUX: PATH=$PATH:$HOME/go_installed/go/bin
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: ''
+          CIBW_TEST_COMMAND: >
+            hugo version
+            hugo env --logLevel debug
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: linux_aarch64_wheels
+          path: ./wheelhouse/*.whl
+          if-no-files-found: error
+
+  macos_x86_64_wheels:
+    strategy:
+      fail-fast: false
+    name: macOS wheels (x86_64)
+    runs-on: macos-latest
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - uses: hynek/build-and-inspect-python-package@v2
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.8
+
+      - name: Set up Go toolchain
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21.x'
+          cache: false
+          check-latest: true
+
+      - uses: pypa/cibuildwheel@v2.16.2
+        with:
+          package-dir: .
+          output-dir: wheelhouse
+        env:
+          CIBW_ARCHS_MACOS: x86_64
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: ''
+          CIBW_TEST_COMMAND: >
+            hugo version
+            hugo env --logLevel debug
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: macos_x86_64_wheels
+          path: ./wheelhouse/*.whl
+          if-no-files-found: error
+
+  macos_arm64_wheels:
+    strategy:
+      fail-fast: false
+    name: macOS wheels (arm64)
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.8
+
+      - name: Set up Go toolchain
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.21.x'
+          cache: false
+          check-latest: true
+
+      - uses: pypa/cibuildwheel@v2.16.2
+        with:
+          package-dir: .
+          output-dir: wheelhouse
+        env:
+          CIBW_ARCHS_MACOS: arm64
+          # These are needed to build arm64 binaries on x86_64 macOS
+          CIBW_ENVIRONMENT_MACOS: >
+            GOARCH="arm64"
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: ''
+          CIBW_TEST_COMMAND: >
+            hugo version
+            hugo env --logLevel debug
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: macos_arm64_wheels
+          path: ./wheelhouse/*.whl
+          if-no-files-found: error
+
+  combined_macos_wheels:
+    name: Combine macOS wheels
+    runs-on: macos-latest
+    needs: [macos_x86_64_wheels, macos_arm64_wheels]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Download macOS wheels for both architectures
+        uses: actions/download-artifact@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Combine macOS wheels
+        run: |
+          mkdir wheelhouse_arm64
+          mkdir wheelhouse_x86_64
+          mkdir wheelhouse_universal2
+          cp macos_x86_64_wheels/*.whl wheelhouse_x86_64/
+          cp macos_arm64_wheels/*.whl wheelhouse_arm64/
+          python -m pip install delocate
+          python scripts/ci/tools/macos/fuse_wheels.py ./wheelhouse_x86_64 ./wheelhouse_arm64 ./wheelhouse_universal2
+
+      - name: Upload combined macOS wheels
+        uses: actions/upload-artifact@v4
+        with:
+          name: combined_macos_wheels
+          path: ./wheelhouse_universal2/*.whl
+          if-no-files-found: error
 
   publish:
-    needs: [dist]
-    name: Publish to PyPI
-    environment: pypi
+    needs: [sdist, windows_wheels, linux_amd64_wheels, linux_aarch64_wheels, combined_macos_wheels]
+    name: Publish to PyPI or TestPyPI
+    environment: release
     permissions:
       id-token: write
     runs-on: ubuntu-latest
     if: github.event_name == 'release' && github.event.action == 'published'
 
     steps:
-      - uses: actions/download-artifact@v4
-        with:
-          name: Packages
-          path: dist
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+
+      - name: Move all artifacts to upload directory
+        run: |
+          mkdir upload
+          mv source_distribution/* windows_wheels/* linux_amd64_wheels/* linux_aarch64_wheels/* combined_macos_wheels/* upload/
 
       - uses: pypa/gh-action-pypi-publish@release/v1
         if: github.event_name == 'release' && github.event.action == 'published'

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -16,13 +16,12 @@ env:
 
 jobs:
   build:
-    name: Wheels (${{ matrix.runs-on }} / Python ${{ matrix.python-version }} / Go ${{ matrix.go-version }})
+    name: Wheels (${{ matrix.runs-on }} / Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.runs-on }}
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.8", "3.12"]
-        go-version: ["1.20.x", "1.21.x"]
         runs-on: [ubuntu-latest, macos-latest, windows-latest]
 
     steps:
@@ -38,7 +37,7 @@ jobs:
       - name: Set up Go toolchain
         uses: actions/setup-go@v5
         with:
-          go-version: ${{ matrix.go-version }}
+          go-version: "1.21.x"
           cache: false
           check-latest: true
 
@@ -54,20 +53,20 @@ jobs:
         uses: actions/cache/restore@v3
         with:
           path: ./hugo_cache/
-          key: ${{ runner.os }}-${{ matrix.go-version }}-hugo-build-cache-${{ hashFiles('**/setup.py', '**/pyproject.toml') }}
+          key: ${{ runner.os }}-hugo-build-cache-${{ hashFiles('**/setup.py', '**/pyproject.toml') }}
 
       - name: Install Python dependencies
         run: python -m pip install build virtualenv nox
 
       - name: Build binary distribution (wheel)
         run: |
-          python -m build --wheel . --outdir dist/
+          python -m build --wheel . --outdir wheelhouse/
 
       - name: Save Hugo builder cache
         uses: actions/cache/save@v3
         with:
           path: ./hugo_cache/
-          key: ${{ runner.os }}-${{ matrix.go-version }}-hugo-build-cache-${{ hashFiles('**/setup.py', '**/pyproject.toml') }}
+          key: ${{ runner.os }}-hugo-build-cache-${{ hashFiles('**/setup.py', '**/pyproject.toml') }}
 
       - name: Test entry points for package
         run: nox -s venv
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,2 +1,3 @@
 include licenses/LICENSE-hugo.txt
-exclude python_hugo/binaries/*
+include python_hugo/binaries/*
+exclude python_hugo/binaries/hugo-*