Support mirroring ptex, scie-jump and provided dists. (#120)

Introduce a `science download` family of commands for mirroring a-scie
binaries and provider distributions to a local directory tree that can
be served up or used directly via `file://` URLs in new `base_url`
configuration options for the corresponding lift manifest entries.

Fixes #113
Fixes #114

.github/workflows/ci.yml

@@ -79,15 +79,15 @@ jobs:
           # native deps like psutil) for our shiv.
           UV_PYTHON_VERSION=cpython-3.12.8-windows-x86_64-none
           uv python install ${UV_PYTHON_VERSION}
-          echo UV_PYTHON_ARGS="--python ${UV_PYTHON_VERSION}" >> ${GITHUB_ENV}
+          echo UV_PYTHON="${UV_PYTHON_VERSION}" >> ${GITHUB_ENV}
       - name: Installing emulators
         if: matrix.docker-platform != ''
         run: docker run --privileged --rm tonistiigi/binfmt --install all
       - name: Checkout Lift
         uses: actions/checkout@v4
       - name: Check Formatting & Lints
         if: matrix.docker-platform == ''
-        run: uv run ${UV_PYTHON_ARGS} dev-cmd ci --skip test
+        run: uv run dev-cmd ci --skip test
       - name: Check Formatting & Lints
         if: matrix.docker-platform != ''
         run: |
@@ -112,7 +112,7 @@ jobs:
           echo SCIE_BASE=C:/tmp/gha/nce >> ${GITHUB_ENV}
       - name: Unit Tests
         if: matrix.docker-platform == ''
-        run: uv run ${UV_PYTHON_ARGS} dev-cmd test -- -vvs
+        run: uv run dev-cmd test -- -vvs
       - name: Unit Tests
         if: matrix.docker-platform != ''
         run: |
@@ -142,7 +142,7 @@ jobs:
             python:3.12-bookworm bash _ci_test.sh
       - name: Build & Package
         if: matrix.docker-platform == ''
-        run: uv run ${UV_PYTHON_ARGS} dev-cmd package
+        run: uv run dev-cmd package
       - name: Build & Package
         if: matrix.docker-platform != ''
         run: |
@@ -161,7 +161,7 @@ jobs:
               "
       - name: Generate Doc Site
         if: matrix.docker-platform == ''
-        run: uv run ${UV_PYTHON_ARGS} dev-cmd doc linkcheck
+        run: uv run dev-cmd doc linkcheck
       - name: Generate Doc Site
         if: matrix.docker-platform != ''
         run: |

.gitignore

@@ -1,10 +1,6 @@
 # The uv venv.
 /.venv/
 
-# Build artifacts dirs.
-/*.egg-info/
-/build/
-
 # Python bytecode.
 __pycache__/
 
@@ -17,5 +13,3 @@ __pycache__/
 # Our Sphinx extension (see: docs/_ext/) dynamically generates doc sources here.
 /docs/_/
 
-# IntelliJ projects.
-.idea/

CHANGES.md

@@ -1,5 +1,12 @@
 # Release Notes
 
+## 0.11.0
+
+This release brings a new `science download` family of commands for downloading `ptex` binaries,
+`scie-jump` binaries and provider distributions for offline use. To complement this, the
+corresponding lift manifest configuration tables now accept a `base_url` option to point to the
+location of these offline downloads.
+
 ## 0.10.1
 
 This release fixes `science` to retry failed HTTP(S) fetches when appropriate.

docs/_ext/sphinx_science/directives.py

@@ -64,7 +64,7 @@ class DirectiveSpec:
     content: StringList = field(default_factory=StringList)
 
     def render_markdown(self) -> str:
-        content_items = [f"```{{{self.name}}} {' '.join(self.args)}"]
+        content_items = [f"```{{{self.name}}} {" ".join(self.args)}"]
         for key, val in self.options.items():
             if val is not None:
                 if isinstance(val, bool):

docs/_ext/sphinx_science/providers.py

@@ -24,10 +24,9 @@ def enumerate_docs(cls, directive_spec: DirectiveSpec) -> Iterator[Doc]:
         )
 
         for provider_info in providers.iter_builtin_providers():
-            slug = provider_info.short_name or provider_info.fully_qualified_name
             yield Doc(
                 id=type_id(provider_info.type),
-                name=slug,
+                name=provider_info.name,
                 directive=dataclasses.replace(
                     directive_spec,
                     args=tuple([provider_info.fully_qualified_name]),

docs/_ext/sphinx_science/toml.py

@@ -116,7 +116,7 @@ def __init__(self, type_: type) -> None:
         super().__init__(
             os.linesep.join(
                 (
-                    f"Raw data type names are not allowed and {type_} has no " "configured name.",
+                    f"Raw data type names are not allowed and {type_} has no configured name.",
                     f"Use @{documented_dataclass.__module__}.documented_dataclass(alias=...) "
                     "to define one.",
                 )

pyproject.toml

@@ -60,6 +60,7 @@ dev = [
     "types-beautifulsoup4",
     "types-docutils",
     "types-psutil",
+    "types-setuptools",
     "types-toml",
     "types-tqdm",
 ]
@@ -95,7 +96,7 @@ check-fmt = ["ruff", "format", "--diff"]
 lint = ["ruff", "check", "--fix"]
 check-lint = ["ruff", "check"]
 
-type-check = ["mypy", "docs/_ext", "science", "scripts", "tests", "test-support"]
+type-check = ["mypy", "docs/_ext", "science", "scripts", "setup.py", "tests", "test-support"]
 
 doc = ["sphinx-build", "-b", "html", "-aEW", "docs", "docs/build/html"]
 linkcheck = ["sphinx-build", "-b", "linkcheck", "-aEW", "docs", "docs/build/linkcheck"]

science/__init__.py

@@ -3,6 +3,6 @@
 
 from packaging.version import Version
 
-__version__ = "0.10.1"
+__version__ = "0.11.0"
 
 VERSION = Version(__version__)

science/a_scie.py

@@ -11,75 +11,78 @@
 
 from packaging.version import Version
 
-from science.fetcher import fetch_and_verify
+from science.fetcher import FetchResult, fetch_and_verify
 from science.hashing import Digest, Fingerprint
-from science.model import File, Ptex, ScieJump, Url
-from science.platform import Platform
+from science.model import Ptex, ScieJump, Url
+from science.platform import CURRENT_PLATFORM, Platform
 
 
 @dataclass(frozen=True)
-class _LoadResult:
-    path: Path
+class LoadResult(FetchResult):
     binary_name: str
 
 
-def _load_project_release(
+def load_project_release(
     project_name: str,
     binary_name: str,
     version: Version | None = None,
     fingerprint: Digest | Fingerprint | None = None,
-    platform: Platform = Platform.current(),
-) -> _LoadResult:
+    platform: Platform = CURRENT_PLATFORM,
+    base_url: Url | None = None,
+) -> LoadResult:
     qualified_binary_name = platform.qualified_binary_name(binary_name)
-    base_url = f"https://github.com/a-scie/{project_name}/releases"
+    root_url = (base_url or f"https://github.com/a-scie/{project_name}/releases").rstrip("/")
     if version:
         version_path = f"download/v{version}"
         ttl = None
     else:
         version_path = "latest/download"
         ttl = timedelta(days=5)
-    path = fetch_and_verify(
-        url=Url(f"{base_url}/{version_path}/{qualified_binary_name}"),
+    result = fetch_and_verify(
+        url=Url(f"{root_url}/{version_path}/{qualified_binary_name}"),
         fingerprint=fingerprint,
         executable=True,
         ttl=ttl,
     )
-    return _LoadResult(path=path, binary_name=qualified_binary_name)
+    return LoadResult(path=result.path, digest=result.digest, binary_name=qualified_binary_name)
 
 
-def jump(specification: ScieJump | None = None, platform: Platform = Platform.current()) -> Path:
+def jump(
+    specification: ScieJump | None = None, platform: Platform = CURRENT_PLATFORM
+) -> LoadResult:
     version = specification.version if specification else None
     fingerprint = specification.digest if specification and specification.digest else None
-    return _load_project_release(
+    base_url = specification.base_url if specification else None
+    return load_project_release(
         project_name="jump",
         binary_name="scie-jump",
         version=version,
         fingerprint=fingerprint,
         platform=platform,
-    ).path
+        base_url=base_url,
+    )
 
 
-def custom_jump(repo_path: Path) -> Path:
+def custom_jump(repo_path: Path) -> LoadResult:
     dist_dir = tempfile.mkdtemp()
     atexit.register(shutil.rmtree, dist_dir, ignore_errors=True)
     subprocess.run(
         args=["cargo", "run", "-p", "package", "--", dist_dir], cwd=repo_path, check=True
     )
-    return Path(dist_dir) / Platform.current().qualified_binary_name("scie-jump")
+    qualified_binary_name = CURRENT_PLATFORM.qualified_binary_name("scie-jump")
+    path = Path(dist_dir) / qualified_binary_name
+    return LoadResult(path=path, digest=Digest.hash(path), binary_name=qualified_binary_name)
 
 
-def ptex(
-    dest_dir: Path, specification: Ptex | None = None, platform: Platform = Platform.current()
-) -> File:
+def ptex(specification: Ptex | None = None, platform: Platform = CURRENT_PLATFORM) -> LoadResult:
     version = specification.version if specification else None
     fingerprint = specification.digest if specification and specification.digest else None
-    result = _load_project_release(
+    base_url = specification.base_url if specification else None
+    return load_project_release(
         project_name="ptex",
-        fingerprint=fingerprint,
         binary_name="ptex",
         version=version,
+        fingerprint=fingerprint,
         platform=platform,
+        base_url=base_url,
     )
-    (dest_dir / result.binary_name).symlink_to(result.path)
-    ptex_key = specification.id if specification and specification.id else "ptex"
-    return File(name=result.binary_name, key=ptex_key, is_executable=True)

science/build_info.py

@@ -18,7 +18,7 @@
 from science.doc import DOC_SITE_URL
 from science.frozendict import FrozenDict
 from science.hashing import Digest, Provenance
-from science.platform import Platform
+from science.platform import CURRENT_PLATFORM
 
 logger = logging.getLogger(__name__)
 
@@ -63,7 +63,7 @@ def to_dict(self, **extra_app_info: Any) -> dict[str, Any]:
             version=__version__,
             url=(
                 f"https://github.com/a-scie/lift/releases/download/v{__version__}/"
-                f"{Platform.current().qualified_binary_name('science')}"
+                f"{CURRENT_PLATFORM.qualified_binary_name("science")}"
             ),
         )
         if self.digest: