Support Josverl/micropython-stubber as stubs repository + Stubs module renovations.

docs/requirements.txt

@@ -7,7 +7,7 @@ babel==2.11.0 ; python_version >= "3.7" and python_version < "3.12"
 bleak-sigspec==0.0.4 ; python_version >= "3.7" and python_version < "3.12"
 bleak-winrt==1.2.0 ; python_version >= "3.7" and python_version < "3.12" and platform_system == "Windows"
 bleak==0.19.4 ; python_version >= "3.7" and python_version < "3.12"
-boltons==19.3.0 ; python_version >= "3.7" and python_version < "3.12"
+boltons==21.0.0 ; python_version >= "3.7" and python_version < "3.12"
 cachier==1.5.4 ; python_version >= "3.7" and python_version < "3.12"
 certifi==2022.9.24 ; python_version >= "3.7" and python_version < "3.12"
 charset-normalizer==2.1.1 ; python_version >= "3.7" and python_version < "3.12"
@@ -16,13 +16,14 @@ colorama==0.4.6 ; python_version >= "3.7" and python_version < "3.12" and sys_pl
 commonmark==0.9.1 ; python_version >= "3.7" and python_version < "3.12"
 dbus-fast==1.73.0 ; python_version >= "3.7" and python_version < "3.12" and platform_system == "Linux"
 dill==0.3.6 ; python_version >= "3.7" and python_version < "3.12"
+distlib==0.3.6 ; python_version >= "3.7" and python_version < "3.12"
 docutils==0.17.1 ; python_version >= "3.7" and python_version < "3.12"
 dpath==1.5.0 ; python_version >= "3.7" and python_version < "3.12"
 gitdb==4.0.9 ; python_version >= "3.7" and python_version < "3.12"
 gitpython==3.1.29 ; python_version >= "3.7" and python_version < "3.12"
 idna==3.4 ; python_version >= "3.7" and python_version < "3.12"
 imagesize==1.4.1 ; python_version >= "3.7" and python_version < "3.12"
-importlib-metadata==5.0.0 ; python_version >= "3.7" and python_version < "3.10"
+importlib-metadata==5.1.0 ; python_version >= "3.7" and python_version < "3.12"
 jinja2==3.1.2 ; python_version >= "3.7" and python_version < "3.12"
 jsonschema==3.2.0 ; python_version >= "3.7" and python_version < "3.12"
 markupsafe==2.1.1 ; python_version >= "3.7" and python_version < "3.12"
@@ -33,6 +34,7 @@ packaging==21.3 ; python_version >= "3.7" and python_version < "3.12"
 pathtools==0.1.2 ; python_version >= "3.7" and python_version < "3.12"
 portalocker==2.6.0 ; python_version >= "3.7" and python_version < "3.12"
 prompt-toolkit==3.0.32 ; python_version >= "3.7" and python_version < "3.12"
+pydantic==1.10.2 ; python_version >= "3.7" and python_version < "3.12"
 pygments==2.13.0 ; python_version >= "3.7" and python_version < "3.12"
 pyobjc-core==8.5.1 ; python_version >= "3.7" and python_version < "3.12" and platform_system == "Darwin"
 pyobjc-framework-cocoa==8.5.1 ; python_version >= "3.7" and python_version < "3.12" and platform_system == "Darwin"
@@ -75,4 +77,4 @@ upydevice==0.3.8 ; python_version >= "3.7" and python_version < "3.12"
 urllib3==1.26.12 ; python_version >= "3.7" and python_version < "3.12"
 watchdog==2.1.9 ; python_version >= "3.7" and python_version < "3.12"
 wcwidth==0.2.5 ; python_version >= "3.7" and python_version < "3.12"
-zipp==3.10.0 ; python_version >= "3.7" and python_version < "3.10"
+zipp==3.10.0 ; python_version >= "3.7" and python_version < "3.12"

micropy/cli.py

@@ -1,8 +1,12 @@
 #!/usr/bin/env python
 
 """MicropyCli Console Entrypoint."""
+from __future__ import annotations
+
 import sys
+from contextlib import suppress
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import click
 import micropy.exceptions as exc
@@ -12,6 +16,9 @@
 from micropy.project import Project, modules
 from questionary import Choice
 
+if TYPE_CHECKING:
+    pass
+
 pass_mpy = click.make_pass_decorator(main.MicroPy, ensure=True)
 
 
@@ -187,7 +194,7 @@ def install(mpy, packages, dev=False, path=None):
 @click.argument("stub_name", required=True)
 @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite Stub if it exists.")
 @pass_mpy
-def add(mpy, stub_name, force=False):
+def add(mpy: main.MicroPy, stub_name, force=False):
     """Add Stubs from package or path.
 
     \b
@@ -208,6 +215,8 @@ def add(mpy, stub_name, force=False):
     mpy.stubs.verbose_log(True)
     proj = mpy.project
     mpy.log.title(f"Adding $[{stub_name}] to stubs")
+    with suppress(exc.StubNotFound):
+        stub_name = mpy.repo.resolve_package(stub_name)
     try:
         stub = mpy.stubs.add(stub_name, force=force)
     except exc.StubNotFound:
@@ -224,15 +233,32 @@ def add(mpy, stub_name, force=False):
 
 @stubs.command()
 @click.argument("query", required=True)
+@click.option(
+    "-s", "--show-outdated", is_flag=True, help="Include outdated versions in search results."
+)
 @pass_mpy
-def search(mpy, query):
+def search(mpy: main.MicroPy, query: str, show_outdated: bool = False):
     """Search available Stubs."""
     mpy.log.title(f"Searching Stub Repositories...")
-    results = mpy.stubs.search_remote(query)
+    installed_stubs = map(str, mpy.stubs._loaded | mpy.stubs._firmware)
+    results = [
+        (r, r.name in installed_stubs)
+        for r in mpy.repo.search(query, include_versions=show_outdated)
+    ]
+    results = sorted(results, key=lambda pkg: pkg[0].name)
+    if not any(results):
+        mpy.log.warn(f"No results found for: $[{query}].")
+        sys.exit(0)
     mpy.log.title(f"Results for $[{query}]:")
+    max_name = max(len(n[0].repo_name) for n in results)
     for pkg, installed in results:
-        name = f"{pkg} $B[(Installed)]" if installed else pkg
-        mpy.log.info(name)
+        pad = max_name - len(pkg.repo_name) + 2
+        pad = pad if (pad % 2 == 0) else pad + 1
+        spacer = "{:>{pad}}".format(f"::", pad=pad)
+        repo_logger = Log.add_logger(f"{pkg.repo_name} {spacer}", "bright_white")
+        name = "{:>{pad}}".format(f"{pkg.name} ($w[{pkg.version}])", pad=pad)
+        name = f"{name} $B[(Installed)]" if installed else name
+        repo_logger.info(name)
 
 
 @stubs.command()

micropy/data/sources.json

@@ -1,6 +1,12 @@
 [
     {
-        "name": "Micropy Stubs",
+        "display_name": "BradenM/micropy-stubs",
+        "name": "micropy-stubs",
         "source": "https://raw.githubusercontent.com/BradenM/micropy-stubs/master/source.json"
+    },
+    {
+        "display_name": "Josverl/micropython-stubs",
+        "name": "micropython-stubs",
+        "source": "https://raw.githubusercontent.com/Josverl/micropython-stubs/main/publish/package_data.jsondb"
     }
 ]

micropy/main.py

@@ -1,26 +1,36 @@
 """Main Module."""
 
+from __future__ import annotations
+
 import tempfile
 from pathlib import Path
+from typing import Optional
 
 from micropy import data, utils
 from micropy.exceptions import PyDeviceError
 from micropy.lib.stubber import process as stubber
 from micropy.logger import Log
 from micropy.project import Project, modules
 from micropy.pyd import DevicePath, MessageHandlers, ProgressStreamConsumer, PyDevice
-from micropy.stubs import StubManager, source
+from micropy.stubs import RepositoryInfo, StubManager, StubRepository
+from pydantic import parse_file_as
 
 
 class MicroPy:
     """Handles App State Management."""
 
     RUN_CHECKS = True
+    repo: StubRepository
+    _stubs: Optional[StubManager] = None
 
     def __init__(self):
         self.log = Log.get_logger("MicroPy")
         self.verbose = True
         self.log.debug("MicroPy Loaded")
+        repo_list = parse_file_as(list[RepositoryInfo], data.REPO_SOURCES)
+        self.repo = StubRepository()
+        for repo_info in repo_list:
+            self.repo = self.repo.add_repository(repo_info)
         if not data.STUB_DIR.exists():
             self.setup()
 
@@ -31,17 +41,17 @@ def setup(self):
         data.FILES.mkdir(exist_ok=True)
         data.STUB_DIR.mkdir()
 
-    @utils.lazy_property
-    def stubs(self):
+    @property
+    def stubs(self) -> StubManager:
         """Primary Stub Manager for MicroPy.
 
         Returns:
             StubManager: StubManager Instance
 
         """
-        repo_list = data.REPO_SOURCES.read_text()
-        repos = source.StubRepo.from_json(repo_list)
-        return StubManager(resource=data.STUB_DIR, repos=repos)
+        if not self._stubs:
+            self._stubs = StubManager(resource=data.STUB_DIR, repos=self.repo)
+        return self._stubs
 
     @utils.lazy_property
     def project(self):

micropy/stubs/__init__.py

@@ -7,6 +7,24 @@
 """
 
 from . import source
+from .manifest import StubsManifest
+from .package import AnyStubPackage, StubPackage
+from .repo import StubRepository
+from .repo_package import StubRepositoryPackage
+from .repositories import MicropyStubPackage, MicropythonStubsManifest, MicropythonStubsPackage
+from .repository_info import RepositoryInfo
 from .stubs import StubManager
 
-__all__ = ["StubManager", "source"]
+__all__ = [
+    "StubManager",
+    "source",
+    "StubsManifest",
+    "StubPackage",
+    "AnyStubPackage",
+    "StubRepository",
+    "MicropyStubPackage",
+    "MicropythonStubsPackage",
+    "MicropythonStubsManifest",
+    "RepositoryInfo",
+    "StubRepositoryPackage",
+]

micropy/stubs/manifest.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import abc
+from typing import Generic
+
+from micropy.stubs.package import AnyStubPackage, StubPackage
+from micropy.stubs.repository_info import RepositoryInfo
+from pydantic import Field
+from pydantic.generics import GenericModel
+from typing_extensions import Annotated
+
+
+class StubsManifest(GenericModel, Generic[AnyStubPackage], abc.ABC):
+
+    class Config:
+        frozen = True
+
+    repository: RepositoryInfo
+    packages: Annotated[frozenset[AnyStubPackage], Field(repr=False)]
+
+    @abc.abstractmethod
+    def resolve_package_url(self, package: StubPackage) -> str:
+        """Resolve package to a stub source."""
+
+    def resolve_package_absolute_name(self, package: StubPackage) -> str:
+        """Resolve package absolute name."""
+        return "/".join([self.repository.name, package.name])
+
+    def resolve_package_versioned_name(self, package: StubPackage) -> str:
+        """Resolve package versioned absolute name."""
+        return "-".join([package.name, package.version])
+
+    def resolve_package_absolute_versioned_name(self, package: StubPackage) -> str:
+        """Resolve package versioned absolute name."""
+        return "-".join([self.resolve_package_absolute_name(package), package.version])

micropy/stubs/package.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import TypeVar
+
+from pydantic import BaseModel
+
+
+class StubPackage(BaseModel):
+
+    class Config:
+        frozen = True
+        allow_population_by_field_name = True
+
+    name: str
+    version: str
+
+    @property
+    def package_name(self) -> str:
+        return self.name
+
+
+AnyStubPackage = TypeVar("AnyStubPackage", bound=StubPackage)