The start of a script to build rst files from collectionized plugins

[_] Build docs for stable
    [x] Parse the collections from a ansibulled deps file
    [x] Download ansible-base that was used in the deps file
    [x] Download collections at those versions
    [x] Install the collections into the ansible-base that was downloaded
    [o] Parse docs out of ansible-base plugins
    [o] Parse docs out of collections
    [_] Construct rst from parsed data
    [_] Construct index.rst from the parsed data
    [_] Output to the right directory structure
[_] Build docs for devel
    [_] Parse the deps from an ansibulled .in file
    [_] Download the latest versions of those collections
    [_] Git clone the latest version of ansible-base
[_] Build docs for a collection
    [_] ?
[_] Build docs for a single plugin
    [_] ?

Changes:

* Add type info for everything ansibulled-docs uses.
* Move code to install collections to its own python module for reuse by
  ansibulled-docs
* Move some constants that are shared between ansibulled-changelog and
  ansibulled-docs into a constants module.
* Move code for untarring tarballs to their own module for reuse with
  ansibulled-docs.
* Create a module that combines venv and sh functionality.  Makes it
  easier to run programs inside of a venv.
* Change changelog.ansible.get_documentable_plugins() to return a frozenset.

ansibulled/ansible_base.py

@@ -0,0 +1,161 @@
+# coding: utf-8
+# Author: Toshio Kuratomi <tkuratom@redhat.com>
+# License: GPLv3+
+# Copyright: Ansible Project, 2020
+"""Functions for working with the ansible-base package."""
+import os
+import re
+from functools import lru_cache
+from typing import TYPE_CHECKING, Any, Dict, List, Union
+from urllib.parse import urljoin
+
+import aiofiles
+import packaging.version as pypiver
+import sh
+
+from .compat import best_get_loop
+from .tarball import pack_tarball
+
+if TYPE_CHECKING:
+    import aiohttp.client
+
+
+#: URL to checkout ansible-base from.
+ANSIBLE_BASE_URL = 'https://github.com/ansible/ansible'
+#: URL to pypi.
+PYPI_SERVER_URL = 'https://test.pypi.org/'
+#: Number of bytes to read or write in one chunk
+CHUNKSIZE = 4096
+
+
+class UnknownVersion(Exception):
+    """Raised when a requested version does not exist."""
+
+
+@lru_cache
+async def checkout_from_git(download_dir: str, repo_url: str = ANSIBLE_BASE_URL) -> str:
+    """
+    Checkout the ansible-base git repo.
+
+    :arg download_dir: Directory to checkout into.
+    :kwarg: repo_url: The url to the git repo.
+    :return: The directory that ansible-base has been checked out to.
+    """
+    loop = best_get_loop()
+    ansible_base_dir = os.path.join(download_dir, 'ansible-base')
+    await loop.run_in_executor(sh.git, 'clone', repo_url, ansible_base_dir)
+
+    return ansible_base_dir
+
+
+class AnsibleBasePyPiClient:
+    """Class to retrieve information about AnsibleBase from Pypi."""
+
+    def __init__(self, aio_session: 'aiohttp.client.ClientSession',
+                 pypi_server_url: str = PYPI_SERVER_URL) -> None:
+        """
+        Initialize the AnsibleBasePypi class.
+
+        :arg aio_session: :obj:`aiohttp.client.ClientSession` to make requests to pypi from.
+        :kwarg pypi_server_url: URL to the pypi server to use.
+        """
+        self.aio_session = aio_session
+        self.pypi_server_url = pypi_server_url
+
+    @lru_cache
+    async def get_info(self) -> Dict[str, Any]:
+        """
+        Retrieve information about the ansible-base package from pypi.
+
+        :returns: The dict which represents the information about the ansible-base package returned
+            from pypi.  To examine the data structure, use::
+
+                curl https://pypi.org/pypi/ansible-base/json| python3 -m json.tool
+        """
+        # Retrieve the ansible-base package info from pypi
+        query_url = urljoin(self.pypi_server_url, 'pypi/ansible-base/json')
+        async with self.aio_session.get(query_url) as response:
+            pkg_info = await response.json()
+        return pkg_info
+
+    async def get_versions(self) -> List[pypiver.Version]:
+        """
+        Get the versions of the ansible-base package on pypi.
+
+        :returns: A list of :pypkg:obj:`packaging.versioning.Version`s
+            for all the versions on pypi, including prereleases.
+        """
+        pkg_info = await self.get_info()
+        versions = [pypiver.Version(r) for r in pkg_info['releases']]
+        versions.sort(reverse=True)
+        return versions
+
+    async def get_latest_version(self) -> pypiver.Version:
+        """
+        Get the latest version of ansible-base uploaded to pypi.
+
+        :return: A :pypkg:obj:`packaging.versioning.Version` object representing the latest version
+            of the package on pypi.  This may be a pre-release.
+        """
+        versions = await self.get_versions()
+        return versions[0]
+
+    async def retrieve(self, ansible_base_version: Union[str, pypiver.Version],
+                       download_dir: str) -> str:
+        """
+        Get the release from pypi.
+
+        :arg ansible_base_version: Version of ansible-base to download.
+        :arg download_dir: Directory to download the tarball to.
+        :returns: The name of the downloaded tarball.
+        """
+        pkg_info = await self.get_info()
+
+        pypi_url = tar_filename = ''
+        for release in pkg_info['releases'][ansible_base_version]:
+            if release['filename'].startswith(f'ansible-base-{ansible_base_version}.tar.'):
+                tar_filename = release['filename']
+                pypi_url = release['url']
+                break
+        else:  # for-else: http://bit.ly/1ElPkyg
+            raise UnknownVersion(f'ansible-base {ansible_base_version} does not'
+                                 ' exist on {pypi_server_url}')
+
+        tar_filename = os.path.join(download_dir, tar_filename)
+        async with self.aio_session.get(pypi_url) as response:
+            async with aiofiles.open(tar_filename, 'wb') as f:
+                # TODO: PY3.8: while chunk := await response.read(CHUNKSIZE):
+                chunk = await response.content.read(CHUNKSIZE)
+                while chunk:
+                    await f.write(chunk)
+                    chunk = await response.content.read(CHUNKSIZE)
+
+        return tar_filename
+
+
+async def create_sdist(dist_dir):
+    ### TODO: Probably should move code that does that into here.
+    pass
+
+
+async def get_ansible_base(aio_session: 'aiohttp.client.ClientSession',
+                           ansible_base_version: str,
+                           tmpdir: str) -> str:
+    """
+    Create an ansible-base directory of the requested version.
+
+    :arg aio_session: :obj:`aiohttp.client.ClientSession` to make http requests with.
+    :arg ansible_base_version: Version of ansible-base to retrieve.
+    :arg tmpdir: Temporary directory use as a scratch area for downloading to and the place that the
+        ansible-base directory should be placed in.
+    """
+    if ansible_base_version == '@devel':
+        install_dir = await checkout_from_git(tmpdir)
+        install_file = await create_sdist(install_dir)
+    else:
+        pypi_client = AnsibleBasePyPiClient(aio_session)
+        if ansible_base_version == '@latest':
+            ansible_base_version: pypiver.Version = await pypi_client.get_latest_version()
+        install_file = await pypi_client.retrieve(ansible_base_version, tmpdir)
+
+    return install_file

ansibulled/build_acd_commands.py

@@ -16,6 +16,7 @@
 import sh
 from jinja2 import Template
 
+from .collections import install_separately, install_together
 from .dependency_files import BuildFile, DepsFile
 from .galaxy import CollectionDownloader
 
@@ -24,15 +25,11 @@
 # Common code
 #
 
-class CollectionFormatError(Exception):
-    pass
-
-
 async def download_collections(deps, download_dir):
     requestors = {}
     async with aiohttp.ClientSession() as aio_session:
+        downloader = CollectionDownloader(aio_session, download_dir)
         for collection_name, version_spec in deps.items():
-            downloader = CollectionDownloader(aio_session, download_dir)
             requestors[collection_name] = asyncio.create_task(
                 downloader.download_latest_matching(collection_name, version_spec))
 
@@ -52,26 +49,6 @@ async def download_collections(deps, download_dir):
 # Single sdist for ansible
 #
 
-async def install_collections_together(version, download_dir, ansible_collections_dir):
-    loop = asyncio.get_running_loop()
-
-    installers = []
-    collection_tarballs = ((p, f) for f in os.listdir(download_dir)
-                           if os.path.isfile(p := os.path.join(download_dir, f)))
-    for pathname, filename in collection_tarballs:
-        namespace, collection, _dummy = filename.split('-', 2)
-        collection_dir = os.path.join(ansible_collections_dir, namespace, collection)
-        # Note: mkdir -p equivalent is okay because we created package_dir ourselves as a directory
-        # that only we can access
-        os.makedirs(collection_dir, mode=0o700, exist_ok=False)
-
-        # If the choice of install tools for galaxy is ever settled upon, we can switch from tar to
-        # using that
-        installers.append(loop.run_in_executor(None, sh.tar, '-xf', pathname, '-C', collection_dir))
-
-    await asyncio.gather(*installers)
-
-
 def copy_boilerplate_files(package_dir):
     gpl_license = pkgutil.get_data('ansibulled.data', 'gplv3.txt')
     with open(os.path.join(package_dir, 'COPYING'), 'wb') as f:
@@ -136,8 +113,7 @@ def build_single_command(args):
         ansible_collections_dir = os.path.join(package_dir, 'ansible_collections')
         os.mkdir(ansible_collections_dir, mode=0o700)
 
-        asyncio.run(install_collections_together(args.acd_version, download_dir,
-                                                 ansible_collections_dir))
+        asyncio.run(install_together(download_dir, ansible_collections_dir))
         write_python_build_files(args.acd_version, '', package_dir)
         make_dist(package_dir, args.dest_dir)
 
@@ -152,45 +128,6 @@ def build_single_command(args):
 #
 
 
-async def install_collections_separately(version, tmp_dir):
-    loop = asyncio.get_running_loop()
-    collection_tarballs = ((p, f) for f in os.listdir(tmp_dir)
-                           if os.path.isfile(p := os.path.join(tmp_dir, f)))
-
-    installers = []
-    collection_dirs = []
-    for pathname, filename in collection_tarballs:
-        namespace, collection, version_ext = filename.split('-', 2)
-        for ext in ('.tar.gz',):
-            # Note: If galaxy allows other archive formats, add their extensions here
-            ext_start = version_ext.find(ext)
-            if ext_start != -1:
-                version = version_ext[:ext_start]
-                break
-        else:
-            raise CollectionFormatError('Collection filename was in an unexpected'
-                                        f' format: {filename}')
-
-        package_dir = os.path.join(tmp_dir, f'ansible-collections-{namespace}.'
-                                   f'{collection}-{version}')
-        os.mkdir(package_dir, mode=0o700)
-        collection_dirs.append(package_dir)
-
-        collection_dir = os.path.join(package_dir, 'ansible_collections', namespace, collection)
-        # Note: this is okay because we created package_dir ourselves as a directory
-        # that only we can access
-        os.makedirs(collection_dir, mode=0o700, exist_ok=False)
-
-        # If the choice of install tools for galaxy is ever settled upon, we can switch from tar to
-        # using that
-        installers.append(loop.run_in_executor(None, sh.tar, '-xf', pathname,
-                                               '-C', collection_dir))
-
-    await asyncio.gather(*installers)
-
-    return collection_dirs
-
-
 async def write_collection_readme(collection_name, package_dir):
     readme_tmpl = Template(pkgutil.get_data('ansibulled.data',
                                             'collection-readme.j2').decode('utf-8'))
@@ -264,8 +201,7 @@ def build_multiple_command(args):
         os.mkdir(download_dir, mode=0o700)
 
         included_versions = asyncio.run(download_collections(deps, download_dir))
-        collection_dirs = asyncio.run(install_collections_separately(args.acd_version,
-                                                                     download_dir))
+        collection_dirs = asyncio.run(install_separately(download_dir))
         asyncio.run(make_collection_dists(args.dest_dir, collection_dirs))
 
         # Create the ansible package that deps on the collections we just wrote

ansibulled/changelog/ansible.py

@@ -7,7 +7,9 @@
 Return Ansible-specific information, like current release or list of documentable plugins.
 """
 
-from typing import Tuple
+from typing import Tuple, FrozenSet
+
+from ..constants import DOCUMENTABLE_PLUGINS
 
 try:
     from ansible import constants as C
@@ -23,16 +25,13 @@
     HAS_ANSIBLE_RELEASE = False
 
 
-def get_documentable_plugins() -> Tuple[str, ...]:
+def get_documentable_plugins() -> FrozenSet[str]:
     """
     Retrieve plugin types that can be documented. Does not include 'module'.
     """
     if HAS_ANSIBLE_CONSTANTS is not None:
-        return C.DOCUMENTABLE_PLUGINS
-    return (
-        'become', 'cache', 'callback', 'cliconf', 'connection', 'httpapi', 'inventory',
-        'lookup', 'netconf', 'shell', 'vars', 'module', 'strategy',
-    )
+        return frozenset(C.DOCUMENTABLE_PLUGINS)
+    return DOCUMENTABLE_PLUGINS
 
 
 def get_ansible_release() -> Tuple[str, str]:

ansibulled/cli/__init__.py

@@ -0,0 +1 @@
+"""Entrypoints to scripts"""