Add support for Windows ARM64. (#85)

This required expanding the recognized platforms as well as a fair bit
of work to support using PBS x86-64 releases for Windows ARM64 under
Prism emulation.

.github/workflows/ci.yml

@@ -23,28 +23,33 @@ jobs:
       matrix:
         # N.B.: macos-12 is the oldest non-deprecated Intel Mac runner and macos-14 is the oldest
         # non-deprecated ARM Mac runner.
-        os: [ubuntu-22.04, linux-arm64, macos-12, macos-14, windows-2022]
+        os: [ ubuntu-22.04, linux-arm64, macos-12, macos-14, windows-2022, windows-arm64 ]
     env:
       SCIENCE_AUTH_API_GITHUB_COM_BEARER: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Setup Python 3.12
-        if: ${{ matrix.os != 'linux-arm64' }}
+        if: matrix.os != 'linux-arm64' && matrix.os != 'windows-arm64'
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - name: Setup Python 3.12
-        if: ${{ matrix.os == 'linux-arm64' }}
+        if: matrix.os == 'linux-arm64'
         run: |
           python3.12 -m venv .venv
           echo "$(pwd)/.venv/bin" >> "${GITHUB_PATH}"
+      - name: Setup Python 3.12
+        if: matrix.os == 'windows-arm64'
+        run: |
+          py -3.12 -m venv .venv
+          echo "$(pwd)/.venv/Scripts" >> "${GITHUB_PATH}"
       - name: Setup Nox
         run: pip install nox
       - name: Checkout Lift
         uses: actions/checkout@v4
       - name: Check Formatting & Lints
         run: nox -e lint
       - name: Configure Windows pytest short tmp dir path
-        if: matrix.os == 'windows-2022'
+        if: matrix.os == 'windows-2022' || matrix.os == 'windows-arm64'
         run: |
           echo PYTEST_ADDOPTS="--basetemp C:\\tmp\\pytest" >> ${GITHUB_ENV}
       - name: Unit Tests

.github/workflows/release.yml

@@ -47,7 +47,7 @@ jobs:
       matrix:
         # N.B.: macos-12 is the oldest non-deprecated Intel Mac runner and macos-14 is the oldest
         # non-deprecated ARM Mac runner.
-        os: [ ubuntu-22.04, linux-arm64, macos-12, macos-14, windows-2022 ]
+        os: [ ubuntu-22.04, linux-arm64, macos-12, macos-14, windows-2022, windows-arm64 ]
     environment: Release
     env:
       SCIENCE_AUTH_API_GITHUB_COM_BEARER: ${{ secrets.GITHUB_TOKEN }}
@@ -58,15 +58,20 @@ jobs:
       discussions: write
     steps:
       - name: Setup Python 3.12
-        if: ${{ matrix.os != 'linux-arm64' }}
+        if: matrix.os != 'linux-arm64' && matrix.os != 'windows-arm64'
         uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - name: Setup Python 3.12
-        if: ${{ matrix.os == 'linux-arm64' }}
+        if: matrix.os == 'linux-arm64'
         run: |
           python3.12 -m venv .venv
           echo "$(pwd)/.venv/bin" >> "${GITHUB_PATH}"
+      - name: Setup Python 3.12
+        if: matrix.os == 'windows-arm64'
+        run: |
+          py -3.12 -m venv .venv
+          echo "$(pwd)/.venv/Scripts" >> "${GITHUB_PATH}"
       - name: Setup Nox
         run: pip install nox
       - name: Checkout lift ${{ needs.determine-tag.outputs.release-tag }}
@@ -85,7 +90,7 @@ jobs:
         with:
           changelog-file: ${{ github.workspace }}/CHANGES.md
           version: ${{ needs.determine-tag.outputs.release-version }}
-          setup-python: ${{ matrix.os != 'linux-arm64' }}
+          setup-python: ${{ matrix.os != 'linux-arm64' && matrix.os != 'windows-arm64' }}
       - name: Create ${{ needs.determine-tag.outputs.release-tag }} Release
         uses: softprops/action-gh-release@v2
         with:

CHANGES.md

@@ -1,5 +1,14 @@
 # Release Notes
 
+# 0.7.0
+
+This release adds support for Windows ARM64. 
+
+> [!NOTE]
+> The `science` binaries shipped for Windows ARM64 are powered by an x86-64 [PBS][PBS] CPython
+> that runs under Windows Prism emulation for x86-64 binaries. As such, you will experience a
+> significantly slower first run when Prism populates its instruction translation caches.
+
 ## 0.6.1
 
 Upgrade the science internal Python distribution to [PBS][PBS] CPython 3.12.5.

README.md

@@ -19,7 +19,7 @@ You'll need to download the correct binary for your system, mark it as executabl
 your $PATH somewhere.
 
 The binaries are released via [GitHub Releases](https://github.com/a-scie/lift/releases)
-for Windows x86_64 and Linux and macOS for both aarch64 and x86_64. For each of these platforms
+for Windows, Linux and macOS for both aarch64 and x86-64. For each of these platforms
 there are two varieties, "thin" and "fat". The "fat" varieties are named `science-fat-*`, include
 a hermetic CPython 3.12 distribution from the [Python Build Standalone]() project and are larger as
 a result. The "thin" varieties have the CPython 3.12 distribution gouged out and are smaller as a

docs/_ext/sphinx_science/directives.py

@@ -143,6 +143,10 @@ class Synthesized(Directive):
             optional_arguments = getattr(doc_gen_directive, "optional_arguments", 0)
 
             def run(self) -> list[nodes.Node]:
+                assert self.state.document.current_source is not None, (
+                    f"We always expect documents we handle to have a source. "
+                    f"This document does not: {self.state.document}"
+                )
                 source = Path(self.state.document.current_source)
                 if generated_toc_node := generated_for.get(source):
                     # N.B.: We use the `env-get-outdated` event below to manually read doctrees

lift.toml

@@ -7,9 +7,12 @@ description = "Ship your interpreted executables using science."
 [[lift.interpreters]]
 id = "cpython"
 provider = "PythonBuildStandalone"
+
+# N.B.: When updating these, update the corresponding PBS_* constants in noxfile.py.
 release = "20240814"
 version = "3.12.5"
 flavor = "install_only_stripped"
+
 # By default science ships as a "thin" scie that fetches CPython 3.12 on first run.
 # We use `science lift --invert-lazy cpython ...` when producing "fat" scies.
 lazy = true
@@ -42,3 +45,4 @@ remove_re = [
 [lift.commands.env.replace]
 SHIV_ROOT = "{scie.bindings}/shiv_root"
 SCIENCE_DOC_LOCAL = "{docsite}"
+__SCIENCE_CURRENT_PLATFORM__ = "{scie.platform}"

noxfile.py

@@ -1,15 +1,23 @@
 # Copyright 2023 Science project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+from __future__ import annotations
+
 import glob
 import hashlib
+import io
 import itertools
 import json
 import os
 import shutil
 import subprocess
+import sys
+import tarfile
+import tempfile
+import urllib.request
 from functools import wraps
 from pathlib import Path
+from textwrap import dedent
 from typing import Any, Callable, Collection, Iterable, TypeVar, cast
 
 import nox
@@ -19,7 +27,32 @@
 nox.options.stop_on_first_error = True
 nox.options.sessions = ["fmt", "lint", "check", "test"]
 
-REQUIRES_PYTHON_VERSION = "3.12"
+# N.B.: Our nox version specifier above admits nox releases that support Python >=3.7, but we set
+# the floor to the oldest Python supported by python.org.
+MIN_NOX_PYTHON = (3, 8)
+MIN_NOX_PYTHON_VER = ".".join(map(str, MIN_NOX_PYTHON))
+
+if sys.version_info[:2] < MIN_NOX_PYTHON:
+    sys.exit(
+        dedent(
+            f"""\
+            This project requires nox running on Python>={MIN_NOX_PYTHON_VER}.
+
+            Your nox is currently running under Python:
+            version info: {sys.version}
+                    path: {sys.executable}
+            """
+        )
+    )
+
+
+# N.B.: When updating these, update the corresponding values for the PythonBuildStandalone provider
+# in lift.toml.
+PBS_RELEASE = "20240814"
+PBS_VERSION = "3.12.5"
+PBS_FLAVOR = "install_only_stripped"
+
+REQUIRES_PYTHON_VERSION = ".".join(PBS_VERSION.split(".")[:2])
 
 PEX_REQUIREMENT = "pex==2.7.0"
 PEX_PEX = f"pex-{hashlib.sha1(PEX_REQUIREMENT.encode('utf-8')).hexdigest()}.pex"
@@ -29,6 +62,25 @@
 LOCK_FILE = BUILD_ROOT / "lock.json"
 
 IS_WINDOWS = os.name == "nt"
+IS_WINDOWS_ARM64 = IS_WINDOWS and subprocess.run(
+    args=["pwsh.exe", "-c", "$Env:PROCESSOR_ARCHITECTURE.ToLower()"],
+    stdout=subprocess.PIPE,
+    text=True,
+).stdout.strip() in ("aarch64", "arm64")
+
+
+def check_lift_manifest(session: Session):
+    session.run(
+        "python",
+        BUILD_ROOT / "scripts" / "check-manifest.py",
+        "--release",
+        PBS_RELEASE,
+        "--version",
+        PBS_VERSION,
+        "--flavor",
+        PBS_FLAVOR,
+        BUILD_ROOT / "lift.toml",
+    )
 
 
 def run_pex(session: Session, script, *args, silent=False, **env) -> Any | None:
@@ -165,11 +217,50 @@ def install_locked_requirements(session: Session, input_reqs: Iterable[Path]) ->
         )
 
 
+def ensure_windows_x86_64_python() -> str:
+    pbs_root = BUILD_ROOT / ".nox" / "PBS" / PBS_RELEASE / PBS_VERSION / PBS_FLAVOR
+    if not pbs_root.exists():
+        url = (
+            "https://github.com/indygreg/python-build-standalone/releases/download/"
+            f"{PBS_RELEASE}/"
+            f"cpython-{PBS_VERSION}+{PBS_RELEASE}-x86_64-pc-windows-msvc-{PBS_FLAVOR}.tar.gz"
+        )
+        with urllib.request.urlopen(f"{url}.sha256") as resp:
+            expected_hash = resp.read().decode().strip()
+        pbs_root.parent.mkdir(parents=True, exist_ok=True)
+        with tempfile.TemporaryDirectory(dir=pbs_root.parent, prefix="PBS-prepare.") as chroot:
+            with urllib.request.urlopen(url) as resp, tempfile.TemporaryFile() as archive:
+                digest = hashlib.sha256()
+                for chunk in iter(lambda: resp.read(io.DEFAULT_BUFFER_SIZE), b""):
+                    archive.write(chunk)
+                    digest.update(chunk)
+                if expected_hash != (actual_hash := digest.hexdigest()):
+                    raise ValueError(
+                        dedent(
+                            f"""\
+                            The PBS archive downloaded from {url} had unexpected contents:
+                            Expected sha256 hash: {expected_hash}
+                            Actual sha256 hash:   {actual_hash}
+                            """
+                        )
+                    )
+                archive.flush()
+                archive.seek(0)
+                with tarfile.open(fileobj=archive) as tf:
+                    tf.extractall(chroot)
+            os.replace(chroot, str(pbs_root))
+    return str(pbs_root / "python" / "python.exe")
+
+
 T = TypeVar("T")
 
 
 def nox_session() -> Callable[[Callable[[Session], T]], Callable[[Session], T]]:
-    return nox.session(python=[REQUIRES_PYTHON_VERSION], reuse_venv=True)
+    # N.B.: We use an x86-64 Python for Windows ARM64 because this is what we ship with via PBS,
+    # and we need to be able to resolve x86-64 compatible requirements (which include native deps
+    # like psutil) for our shiv.
+    python = ensure_windows_x86_64_python() if IS_WINDOWS_ARM64 else REQUIRES_PYTHON_VERSION
+    return nox.session(python=[python], reuse_venv=True)
 
 
 @nox_session()
@@ -197,8 +288,8 @@ def wrapper(session: Session) -> T:
                     requirements.append(session_reqs)
             if include_project:
                 requirements.append(BUILD_ROOT / "requirements.txt")
-
-            install_locked_requirements(session, input_reqs=requirements)
+            if requirements:
+                install_locked_requirements(session, input_reqs=requirements)
             return func(session)
 
         return nox_session()(wrapper)
@@ -237,6 +328,8 @@ def lint(session: Session) -> None:
 
 @python_session(include_project=True, extra_reqs=["doc", "test"])
 def check(session: Session) -> None:
+    check_lift_manifest(session)
+    session.run("mypy", "--python-version", ".".join(map(str, MIN_NOX_PYTHON)), "noxfile.py")
     session.run(
         "mypy", "--python-version", REQUIRES_PYTHON_VERSION, *PATHS_TO_CHECK, *session.posargs
     )
@@ -261,6 +354,9 @@ def create_zipapp(session: Session) -> Path:
                 str(BUILD_ROOT / "requirements.windows-amd64.lock.txt"),
                 external=True,
             )
+            session.run(
+                str(venv_dir / "Scripts" / "python.exe"), "-m", "pip", "uninstall", "--yes", "pip"
+            )
             site_packages = str(venv_dir / "Lib" / "site-packages")
         else:
             run_pex(

science/__init__.py

@@ -3,6 +3,6 @@
 
 from packaging.version import Version
 
-__version__ = "0.6.1"
+__version__ = "0.7.0"
 
 VERSION = Version(__version__)

science/commands/doc.py

@@ -15,6 +15,7 @@
 from datetime import datetime
 from functools import cache
 from pathlib import Path, PurePath
+from typing import Any
 
 import psutil
 
@@ -179,7 +180,7 @@ def launch(
     env = {**os.environ, "PYTHONUNBUFFERED": "1"}
     with log.open("w") as fp:
         # Not proper daemonization, but good enough.
-        daemon_kwargs = (
+        daemon_kwargs: dict[str, Any] = (
             {
                 # The subprocess.{DETACHED_PROCESS,CREATE_NEW_PROCESS_GROUP} attributes are only
                 # defined on Windows.
@@ -188,8 +189,11 @@ def launch(
                     | subprocess.CREATE_NEW_PROCESS_GROUP  # type: ignore[attr-defined]
                 )
             }
-            if Platform.current() is Platform.Windows_x86_64
-            else {"preexec_fn": os.setsid}
+            if Platform.current() in (Platform.Windows_aarch64, Platform.Windows_x86_64)
+            else {
+                # The os.setsid function is not available on Windows.
+                "preexec_fn": os.setsid  # type: ignore[attr-defined]
+            }
         )
         process = subprocess.Popen(
             args=[sys.executable, "-m", "http.server", str(port)],

science/commands/lift.py

@@ -89,6 +89,7 @@ class LiftConfig:
     include_provenance: bool = False
     app_info: tuple[AppInfo, ...] = ()
     app_name: str | None = None
+    platforms: tuple[Platform, ...] = ()
 
 
 def export_manifest(

science/exe.py

@@ -495,6 +495,14 @@ def _list(emit_json: bool) -> None:
         """
     ),
 )
+@click.option(
+    "--platform",
+    "platforms",
+    type=Platform.parse,
+    multiple=True,
+    default=[],
+    help="Override any configured platforms and target these platforms instead.",
+)
 @click.pass_context
 def _lift(
     ctx: click.Context,
@@ -503,6 +511,7 @@ def _lift(
     include_provenance: bool,
     app_name: str | None,
     app_info: list[AppInfo],
+    platforms: list[Platform],
 ) -> None:
     # N.B.: Help is defined above in the _lift group decorator since it's a dynamic string.
     ctx.obj = LiftConfig(
@@ -511,6 +520,7 @@ def _lift(
         include_provenance=include_provenance or bool(app_info),
         app_info=tuple(app_info),
         app_name=app_name,
+        platforms=tuple(platforms),
     )
 
 
@@ -581,7 +591,9 @@ def export(
     application = parse_application(lift_config, config)
     platform_info = PlatformInfo.create(application, use_suffix=use_platform_suffix)
     with temporary_directory(cleanup=True) as td:
-        for _, manifest_path in lift.export_manifest(lift_config, application, dest_dir=td):
+        for _, manifest_path in lift.export_manifest(
+            lift_config, application, dest_dir=td, platforms=lift_config.platforms
+        ):
             lift_manifest = dest_dir / (
                 manifest_path.relative_to(td) if platform_info.use_suffix else manifest_path.name
             )
@@ -677,7 +689,7 @@ def _build(
     application = parse_application(lift_config, config)
     platform_info = PlatformInfo.create(application, use_suffix=use_platform_suffix)
 
-    platforms = application.platforms
+    platforms = lift_config.platforms or application.platforms
     if use_jump and use_platform_suffix:
         logger.warning(f"Cannot use a custom scie jump build with a multi-platform configuration.")
         logger.warning(