diff --git a/antsibull/cli/doc_commands/plugin.py b/antsibull/cli/doc_commands/plugin.py index 0ec156a4..1328b5fa 100644 --- a/antsibull/cli/doc_commands/plugin.py +++ b/antsibull/cli/doc_commands/plugin.py @@ -15,6 +15,7 @@ from ... import app_context from ...augment_docs import augment_docs from ...compat import asyncio_run +from ...docs_parsing import AnsibleCollectionMetadata from ...docs_parsing.fqcn import get_fqcn_parts, is_fqcn from ...jinja2.environment import doc_environment from ...logging import log @@ -103,7 +104,7 @@ def generate_docs() -> int: error_tmpl = env.get_template('plugin-error.rst.j2') asyncio_run(write_rst( - '.'.join([namespace, collection]), plugin, plugin_type, + '.'.join([namespace, collection]), AnsibleCollectionMetadata.empty(), plugin, plugin_type, plugin_info, errors, plugin_tmpl, error_tmpl, '', path_override=output_path)) flog.debug('Finished writing plugin docs') diff --git a/antsibull/cli/doc_commands/stable.py b/antsibull/cli/doc_commands/stable.py index 4546d21b..c5569999 100644 --- a/antsibull/cli/doc_commands/stable.py +++ b/antsibull/cli/doc_commands/stable.py @@ -268,10 +268,12 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner], """ # Get the info from the plugins - plugin_info = asyncio_run(get_ansible_plugin_info( + plugin_info, collection_metadata = asyncio_run(get_ansible_plugin_info( venv, collection_dir, collection_names=collection_names)) - flog.notice('Finished parsing info from plugins') + flog.notice('Finished parsing info from plugins and collections') # flog.fields(plugin_info=plugin_info).debug('Plugin data') + # flog.fields( + # collection_metadata=collection_metadata).debug('Collection metadata') """ # Turn these into some sort of decorator that will choose to dump or load the values @@ -311,21 +313,24 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner], """ plugin_contents = get_plugin_contents(plugin_info, nonfatal_errors) - collection_info = get_collection_contents(plugin_contents) + collection_to_plugin_info = get_collection_contents(plugin_contents) flog.debug('Finished getting collection data') # Only build top-level index if requested if create_indexes: - asyncio_run(output_collection_index(collection_info, dest_dir)) + asyncio_run(output_collection_index(collection_to_plugin_info, dest_dir)) flog.notice('Finished writing collection index') asyncio_run(output_plugin_indexes(plugin_contents, dest_dir)) flog.notice('Finished writing plugin indexes') - asyncio_run(output_indexes(collection_info, dest_dir, squash_hierarchy=squash_hierarchy)) + asyncio_run(output_indexes(collection_to_plugin_info, dest_dir, + collection_metadata=collection_metadata, + squash_hierarchy=squash_hierarchy)) flog.notice('Finished writing indexes') - asyncio_run(output_all_plugin_rst(collection_info, plugin_info, + asyncio_run(output_all_plugin_rst(collection_to_plugin_info, plugin_info, nonfatal_errors, dest_dir, + collection_metadata=collection_metadata, squash_hierarchy=squash_hierarchy)) flog.debug('Finished writing plugin docs') diff --git a/antsibull/data/docsite/plugin.rst.j2 b/antsibull/data/docsite/plugin.rst.j2 index 0df49509..3917b1a3 100644 --- a/antsibull/data/docsite/plugin.rst.j2 +++ b/antsibull/data/docsite/plugin.rst.j2 @@ -52,7 +52,7 @@ the same module name. {% else %} .. note:: - This plugin is part of the `@{collection}@ collection `_. + This plugin is part of the `@{collection}@ collection `_{% if collection_version %} (version @{ collection_version }@){% endif %}. To install it use: :code:`ansible-galaxy collection install @{collection}@`. diff --git a/antsibull/data/docsite/plugins_by_collection.rst.j2 b/antsibull/data/docsite/plugins_by_collection.rst.j2 index 557a78e3..9eb2c7a4 100644 --- a/antsibull/data/docsite/plugins_by_collection.rst.j2 +++ b/antsibull/data/docsite/plugins_by_collection.rst.j2 @@ -2,22 +2,29 @@ .. _plugins_in_@{collection_name}@: -Plugin Index -============ +@{collection_name.title()}@ +@{ '=' * (collection_name | length) }@ -These are the plugins in the @{collection_name}@ collection +{% if collection_version %} +Collection version @{ collection_version }@ +{% endif %} .. toctree:: :maxdepth: 1 +Plugin Index +------------ + +These are the plugins in the @{collection_name}@ collection + {% for category, plugins in plugin_maps.items() | sort %} {% if category == 'module' %} Modules -------- +~~~~~~~ {% else %} @{ category | capitalize }@ Plugins -@{ '-' * ((category | length) + 8) }@ +@{ '~' * ((category | length) + 8) }@ {% endif %} {% for name, desc in plugins.items() | sort %} diff --git a/antsibull/docs_parsing/__init__.py b/antsibull/docs_parsing/__init__.py index 2dca2807..1ccbb5f1 100644 --- a/antsibull/docs_parsing/__init__.py +++ b/antsibull/docs_parsing/__init__.py @@ -62,3 +62,19 @@ def _get_environment(collection_dir: t.Optional[str]) -> t.Dict[str, str]: if env_var in os.environ: env[env_var] = os.environ[env_var] return env + + +class AnsibleCollectionMetadata: + path: str + version: t.Optional[str] + + def __init__(self, path: str, version: t.Optional[str]): + self.path = path + self.version = version + + def __repr__(self): + return 'AnsibleCollectionMetadata({0}, {1})'.format(repr(self.path), repr(self.version)) + + @classmethod + def empty(cls, path='.'): + return cls(path=path, version=None) diff --git a/antsibull/docs_parsing/ansible_doc.py b/antsibull/docs_parsing/ansible_doc.py index d4cbad3b..37f1ae5e 100644 --- a/antsibull/docs_parsing/ansible_doc.py +++ b/antsibull/docs_parsing/ansible_doc.py @@ -6,6 +6,7 @@ import asyncio import json import sys +import os import traceback import typing as t from concurrent.futures import ThreadPoolExecutor @@ -18,7 +19,7 @@ from ..logging import log from ..vendored.json_utils import _filter_non_json_lines from .fqcn import get_fqcn_parts -from . import _get_environment, ParsingError +from . import _get_environment, ParsingError, AnsibleCollectionMetadata if t.TYPE_CHECKING: from ..venv import VenvRunner, FakeVenvRunner @@ -158,10 +159,56 @@ async def _get_plugin_info(plugin_type: str, ansible_doc: 'sh.Command', return results +def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'], + env: t.Dict[str, str], + collection_names: t.Optional[t.List[str]] = None, + ) -> t.Dict[str, AnsibleCollectionMetadata]: + collection_metadata = {} + + # Obtain ansible.builtin version + if collection_names is None or 'ansible.builtin' in collection_names: + venv_ansible = venv.get_command('ansible') + ansible_version_cmd = venv_ansible('--version', _env=env) + raw_result = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape') + path: t.Optional[str] = None + version: t.Optional[str] = None + for line in raw_result.splitlines(): + if line.strip().startswith('ansible python module location'): + path = line.split('=', 2)[1].strip() + if line.startswith('ansible '): + version = line[len('ansible '):] + collection_metadata['ansible.builtin'] = AnsibleCollectionMetadata( + path=path, version=version) + + # Obtain collection versions + venv_ansible_galaxy = venv.get_command('ansible-galaxy') + ansible_collection_list_cmd = venv_ansible_galaxy('collection', 'list', _env=env) + raw_result = ansible_collection_list_cmd.stdout.decode('utf-8', errors='surrogateescape') + current_base_path = None + for line in raw_result.splitlines(): + parts = line.split() + if len(parts) >= 2: + if parts[0] == '#': + current_base_path = parts[1] + else: + collection_name = parts[0] + version = parts[1] + if '.' in collection_name: + if collection_names is None or collection_name in collection_names: + namespace, name = collection_name.split('.', 2) + collection_metadata[collection_name] = AnsibleCollectionMetadata( + path=os.path.join(current_base_path, namespace, name), + version=None if version == '*' else version) + + return collection_metadata + + async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], collection_dir: t.Optional[str], collection_names: t.Optional[t.List[str]] = None - ) -> t.Dict[str, t.Dict[str, t.Any]]: + ) -> t.Tuple[ + t.Mapping[str, t.Mapping[str, t.Any]], + t.Mapping[str, t.Any]]: """ Retrieve information about all of the Ansible Plugins. @@ -171,12 +218,14 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], search path for Ansible. :arg collection_names: Optional list of collections. If specified, will only collect information for plugins in these collections. - :returns: A nested directory structure that looks like:: + :returns: An tuple. The first component is a nested directory structure that looks like: - plugin_type: - plugin_name: # Includes namespace and collection. - {information from ansible-doc --json. See the ansible-doc documentation for more - info.} + plugin_type: + plugin_name: # Includes namespace and collection. + {information from ansible-doc --json. See the ansible-doc documentation + for more info.} + + The second component is a Mapping of collection names to metadata. """ flog = mlog.fields(func='get_ansible_plugin_info') flog.debug('Enter') @@ -246,5 +295,8 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], # done so, we want to then fail by raising one of the exceptions. raise ParsingError('Parsing of plugins failed') + flog.debug('Retrieving collection metadata') + collection_metadata = get_collection_metadata(venv, env, collection_names) + flog.debug('Leave') - return plugin_map + return (plugin_map, collection_metadata) diff --git a/antsibull/docs_parsing/ansible_internal.py b/antsibull/docs_parsing/ansible_internal.py index 985ad7d1..0acc6e9c 100644 --- a/antsibull/docs_parsing/ansible_internal.py +++ b/antsibull/docs_parsing/ansible_internal.py @@ -11,7 +11,7 @@ from ..logging import log from ..utils.get_pkg_data import get_antsibull_data from ..vendored.json_utils import _filter_non_json_lines -from . import _get_environment +from . import _get_environment, AnsibleCollectionMetadata if t.TYPE_CHECKING: from ..venv import VenvRunner, FakeVenvRunner @@ -23,7 +23,9 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], collection_dir: t.Optional[str], collection_names: t.Optional[t.List[str]] = None - ) -> t.Dict[str, t.Dict[str, t.Any]]: + ) -> t.Tuple[ + t.Mapping[str, t.Mapping[str, t.Any]], + t.Mapping[str, t.Any]]: """ Retrieve information about all of the Ansible Plugins. @@ -33,12 +35,14 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], search path for Ansible. :arg collection_names: Optional list of collections. If specified, will only collect information for plugins in these collections. - :returns: A nested directory structure that looks like:: + :returns: An tuple. The first component is a nested directory structure that looks like: - plugin_type: - plugin_name: # Includes namespace and collection. - {information from ansible-doc --json. See the ansible-doc documentation for more - info.} + plugin_type: + plugin_name: # Includes namespace and collection. + {information from ansible-doc --json. See the ansible-doc documentation + for more info.} + + The second component is a Mapping of collection names to metadata. """ flog = mlog.fields(func='get_ansible_plugin_info') flog.debug('Enter') @@ -72,7 +76,11 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], plugin_log.fields(error=plugin_data['error']).error( 'Error while extracting documentation. Will not document this plugin.') - # TODO: use result['collections'] + collection_metadata = {} + for collection_name, collection_data in result['collections'].items(): + collection_metadata[collection_name] = AnsibleCollectionMetadata( + path=collection_data['path'], + version=collection_data.get('version')) flog.debug('Leave') - return plugin_map + return (plugin_map, collection_metadata) diff --git a/antsibull/docs_parsing/parsing.py b/antsibull/docs_parsing/parsing.py index cf066f5b..2fadf243 100644 --- a/antsibull/docs_parsing/parsing.py +++ b/antsibull/docs_parsing/parsing.py @@ -20,7 +20,9 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], collection_dir: t.Optional[str], collection_names: t.Optional[t.List[str]] = None - ) -> t.Dict[str, t.Dict[str, t.Any]]: + ) -> t.Tuple[ + t.Mapping[str, t.Mapping[str, t.Any]], + t.Mapping[str, t.Any]]: """ Retrieve information about all of the Ansible Plugins. @@ -30,12 +32,15 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], search path for Ansible. :arg collection_names: Optional list of collections. If specified, will only collect information for plugins in these collections. - :returns: A nested directory structure that looks like:: + :returns: An tuple. The first component is a nested directory structure that looks like: + + plugin_type: + plugin_name: # Includes namespace and collection. + {information from ansible-doc --json. See the ansible-doc documentation + for more info.} + + The second component is a Mapping of collection names to metadata. - plugin_type: - plugin_name: # Includes namespace and collection. - {information from ansible-doc --json. See the ansible-doc documentation for more - info.} """ lib_ctx = app_context.lib_ctx.get() diff --git a/antsibull/write_docs.py b/antsibull/write_docs.py index 2e4724d9..62a3dde7 100644 --- a/antsibull/write_docs.py +++ b/antsibull/write_docs.py @@ -16,6 +16,7 @@ from . import app_context from .jinja2.environment import doc_environment from .logging import log +from .docs_parsing import AnsibleCollectionMetadata mlog = log.fields(mod=__name__) @@ -33,7 +34,8 @@ PluginCollectionInfoT = t.Mapping[str, t.Mapping[str, t.Mapping[str, str]]] -async def write_rst(collection_name: str, plugin_short_name: str, plugin_type: str, +async def write_rst(collection_name: str, collection_meta: AnsibleCollectionMetadata, + plugin_short_name: str, plugin_type: str, plugin_record: t.Dict[str, t.Any], nonfatal_errors: t.Sequence[str], plugin_tmpl: Template, error_tmpl: Template, dest_dir: str, path_override: t.Optional[str] = None, @@ -42,6 +44,7 @@ async def write_rst(collection_name: str, plugin_short_name: str, plugin_type: s Write the rst page for one plugin. :arg collection_name: Dotted colection name. + :arg collection_meta: Collection metadata object. :arg plugin_short_name: short name for the plugin. :arg plugin_type: The type of the plugin. (module, inventory, etc) :arg plugin_record: The record for the plugin. doc, examples, and return are the @@ -72,6 +75,7 @@ async def write_rst(collection_name: str, plugin_short_name: str, plugin_type: s plugin_contents = error_tmpl.render( plugin_type=plugin_type, plugin_name=plugin_name, collection=collection_name, + collection_version=collection_meta.version, nonfatal_errors=nonfatal_errors) else: if nonfatal_errors: @@ -82,6 +86,7 @@ async def write_rst(collection_name: str, plugin_short_name: str, plugin_type: s plugin_name=plugin_name) plugin_contents = plugin_tmpl.render( collection=collection_name, + collection_version=collection_meta.version, plugin_type=plugin_type, plugin_name=plugin_name, doc=plugin_record['doc'], @@ -108,20 +113,22 @@ async def write_rst(collection_name: str, plugin_short_name: str, plugin_type: s flog.debug('Leave') -async def output_all_plugin_rst(collection_info: CollectionInfoT, +async def output_all_plugin_rst(collection_to_plugin_info: CollectionInfoT, plugin_info: t.Dict[str, t.Any], nonfatal_errors: PluginErrorsT, dest_dir: str, + collection_metadata: t.Mapping[str, AnsibleCollectionMetadata], squash_hierarchy: bool = False) -> None: """ Output rst files for each plugin. - :arg collection_info: Mapping of collection_name to Mapping of plugin_type to Mapping of - collection_name to short_description. + :arg collection_to_plugin_info: Mapping of collection_name to Mapping of plugin_type to Mapping + of plugin_name to short_description. :arg plugin_info: Documentation information for all of the plugins. :arg nonfatal_errors: Mapping of plugins to nonfatal errors. Using this to note on the docs pages when documentation wasn't formatted such that we could use it. :arg dest_dir: The directory to place the documentation in. + :arg collection_metadata: Dictionary mapping collection names to collection metadata objects. :arg squash_hierarchy: If set to ``True``, no directory hierarchy will be used. Undefined behavior if documentation for multiple collections are created. @@ -135,12 +142,14 @@ async def output_all_plugin_rst(collection_info: CollectionInfoT, writers = [] lib_ctx = app_context.lib_ctx.get() async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool: - for collection_name, plugins_by_type in collection_info.items(): + for collection_name, plugins_by_type in collection_to_plugin_info.items(): for plugin_type, plugins in plugins_by_type.items(): for plugin_short_name, dummy_ in plugins.items(): plugin_name = '.'.join((collection_name, plugin_short_name)) writers.append(await pool.spawn( - write_rst(collection_name, plugin_short_name, plugin_type, + write_rst(collection_name, + collection_metadata[collection_name], + plugin_short_name, plugin_type, plugin_info[plugin_type].get(plugin_name), nonfatal_errors[plugin_type][plugin_name], plugin_tmpl, error_tmpl, dest_dir, squash_hierarchy=squash_hierarchy))) @@ -191,7 +200,8 @@ async def write_plugin_type_index(plugin_type: str, async def write_plugin_lists(collection_name: str, plugin_maps: t.Mapping[str, t.Mapping[str, str]], template: Template, - dest_dir: str) -> None: + dest_dir: str, + collection_meta: AnsibleCollectionMetadata) -> None: """ Write an index page for each collection. @@ -200,10 +210,12 @@ async def write_plugin_lists(collection_name: str, :arg plugin_maps: Mapping of plugin_type to Mapping of plugin_name to short_description. :arg template: A template to render the collection index. :arg dest_dir: The destination directory to output the index into. + :arg collection_meta: Metadata for the collection. """ index_contents = template.render( collection_name=collection_name, - plugin_maps=plugin_maps) + plugin_maps=plugin_maps, + collection_version=collection_meta.version) # This is only safe because we made sure that the top of the directory tree we're writing to # (docs/docsite/rst) is only writable by us. @@ -214,13 +226,13 @@ async def write_plugin_lists(collection_name: str, await f.write(index_contents) -async def output_collection_index(collection_info: CollectionInfoT, +async def output_collection_index(collection_to_plugin_info: CollectionInfoT, dest_dir: str) -> None: """ Generate top-level collection index page for the collections. - :arg collection_info: Mapping of collection_name to Mapping of plugin_type to Mapping of - collection_name to short_description. + :arg collection_to_plugin_info: Mapping of collection_name to Mapping of plugin_type to Mapping + of plugin_name to short_description. :arg dest_dir: The directory to place the documentation in. """ flog = mlog.fields(func='output_collection_index') @@ -237,7 +249,7 @@ async def output_collection_index(collection_info: CollectionInfoT, # (docs/docsite/rst) is only writable by us. os.makedirs(collection_toplevel, mode=0o755, exist_ok=True) - await write_collection_list(collection_info.keys(), collection_list_tmpl, + await write_collection_list(collection_to_plugin_info.keys(), collection_list_tmpl, collection_toplevel) flog.debug('Leave') @@ -281,15 +293,18 @@ async def output_plugin_indexes(plugin_info: PluginCollectionInfoT, flog.debug('Leave') -async def output_indexes(collection_info: CollectionInfoT, +async def output_indexes(collection_to_plugin_info: CollectionInfoT, dest_dir: str, - squash_hierarchy: bool = False) -> None: + collection_metadata: t.Mapping[str, AnsibleCollectionMetadata], + squash_hierarchy: bool = False, + ) -> None: """ Generate collection-level index pages for the collections. - :arg collection_info: Mapping of collection_name to Mapping of plugin_type to Mapping of - collection_name to short_description. + :arg collection_to_plugin_info: Mapping of collection_name to Mapping of plugin_type to Mapping + of plugin_name to short_description. :arg dest_dir: The directory to place the documentation in. + :arg collection_metadata: Dictionary mapping collection names to collection metadata objects. :arg squash_hierarchy: If set to ``True``, no directory hierarchy will be used. Undefined behavior if documentation for multiple collections are created. @@ -297,6 +312,9 @@ async def output_indexes(collection_info: CollectionInfoT, flog = mlog.fields(func='output_indexes') flog.debug('Enter') + if collection_metadata is None: + collection_metadata = {} + env = doc_environment(('antsibull.data', 'docsite')) # Get the templates collection_plugins_tmpl = env.get_template('plugins_by_collection.rst.j2') @@ -315,14 +333,14 @@ async def output_indexes(collection_info: CollectionInfoT, collection_toplevel = dest_dir async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool: - for collection_name, plugin_maps in collection_info.items(): + for collection_name, plugin_maps in collection_to_plugin_info.items(): if not squash_hierarchy: collection_dir = os.path.join(collection_toplevel, *(collection_name.split('.'))) else: collection_dir = collection_toplevel writers.append(await pool.spawn( write_plugin_lists(collection_name, plugin_maps, collection_plugins_tmpl, - collection_dir))) + collection_dir, collection_metadata[collection_name]))) await asyncio.gather(*writers)