From 89a4536984e1180b55962e7ab871b02365eba90a Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 3 Oct 2020 22:34:28 +0200 Subject: [PATCH] Add collection versions. --- antsibull/cli/doc_commands/plugin.py | 2 +- antsibull/cli/doc_commands/stable.py | 22 +++++---- antsibull/data/docsite/plugin.rst.j2 | 2 +- .../data/docsite/plugins_by_collection.rst.j2 | 3 ++ antsibull/docs_parsing/__init__.py | 18 +++++++ antsibull/docs_parsing/ansible_doc.py | 47 +++++++++++++++---- antsibull/docs_parsing/ansible_internal.py | 17 +++---- antsibull/docs_parsing/parsing.py | 10 ++-- antsibull/write_docs.py | 33 ++++++++++--- 9 files changed, 110 insertions(+), 44 deletions(-) diff --git a/antsibull/cli/doc_commands/plugin.py b/antsibull/cli/doc_commands/plugin.py index 0ec156a4c..45333d9df 100644 --- a/antsibull/cli/doc_commands/plugin.py +++ b/antsibull/cli/doc_commands/plugin.py @@ -103,7 +103,7 @@ def generate_docs() -> int: error_tmpl = env.get_template('plugin-error.rst.j2') asyncio_run(write_rst( - '.'.join([namespace, collection]), plugin, plugin_type, + '.'.join([namespace, collection]), None, plugin, plugin_type, plugin_info, errors, plugin_tmpl, error_tmpl, '', path_override=output_path)) flog.debug('Finished writing plugin docs') diff --git a/antsibull/cli/doc_commands/stable.py b/antsibull/cli/doc_commands/stable.py index 563edc9d1..0372b87ce 100644 --- a/antsibull/cli/doc_commands/stable.py +++ b/antsibull/cli/doc_commands/stable.py @@ -248,24 +248,26 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner], create_index = (collection_names is None) # Get the info from the plugins - plugin_info = asyncio_run(get_ansible_plugin_info( + collection_docs = asyncio_run(get_ansible_plugin_info( venv, collection_dir, collection_names=collection_names)) - flog.notice('Finished parsing info from plugins') - # flog.fields(plugin_info=plugin_info).debug('Plugin data') + flog.notice('Finished parsing info from plugins and collections') + # flog.fields(plugin_info=collection_docs.plugins).debug('Plugin data') + # flog.fields( + # collection_versions=collection_docs.collection_versions).debug('Collection versions') """ # Turn these into some sort of decorator that will choose to dump or load the values # if a command line arg is specified. with open('dump_raw_plugin_info.json', 'w') as f: import json - json.dump(plugin_info, f) - flog.debug('Finished dumping raw plugin_info') + json.dump(collection_docs.plugins, f) + flog.debug('Finished dumping raw collection_docs.plugins') with open('dump_formatted_plugin_info.json', 'r') as f: import json - plugin_info = json.load(f) + collection_docs.plugins = json.load(f) """ - plugin_info, nonfatal_errors = asyncio_run(normalize_all_plugin_info(plugin_info)) + plugin_info, nonfatal_errors = asyncio_run(normalize_all_plugin_info(collection_docs.plugins)) flog.fields(errors=len(nonfatal_errors)).notice('Finished data validation') augment_docs(plugin_info) flog.notice('Finished calculating new data') @@ -300,12 +302,14 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner], asyncio_run(output_plugin_indexes(collection_info, dest_dir)) flog.notice('Finished writing plugin indexes') - asyncio_run(output_indexes(collection_info, dest_dir, squash_hierarchy=squash_hierarchy)) + asyncio_run(output_indexes(collection_info, dest_dir, squash_hierarchy=squash_hierarchy, + collection_versions=collection_docs.collection_versions)) flog.notice('Finished writing indexes') asyncio_run(output_all_plugin_rst(collection_info, plugin_info, nonfatal_errors, dest_dir, - squash_hierarchy=squash_hierarchy)) + squash_hierarchy=squash_hierarchy, + collection_versions=collection_docs.collection_versions)) flog.debug('Finished writing plugin docs') diff --git a/antsibull/data/docsite/plugin.rst.j2 b/antsibull/data/docsite/plugin.rst.j2 index 6c785dc9f..67a991b98 100644 --- a/antsibull/data/docsite/plugin.rst.j2 +++ b/antsibull/data/docsite/plugin.rst.j2 @@ -52,7 +52,7 @@ the same module name. {% else %} .. note:: - This plugin is part of the `@{collection}@ collection `_. + This plugin is part of the `@{collection}@ collection `_{% if collection_version %} (version @{ collection_version }@){% endif %}. To install it use: :code:`ansible-galaxy collection install @{collection}@`. diff --git a/antsibull/data/docsite/plugins_by_collection.rst.j2 b/antsibull/data/docsite/plugins_by_collection.rst.j2 index 0d8384f7c..84cbc1d2c 100644 --- a/antsibull/data/docsite/plugins_by_collection.rst.j2 +++ b/antsibull/data/docsite/plugins_by_collection.rst.j2 @@ -6,6 +6,9 @@ Plugin Index ============ These are the plugins in the @{collection_name}@ collection +{% if collection_version %} +(version @{ collection_version }@) +{% endif %} .. toctree:: :maxdepth: 1 diff --git a/antsibull/docs_parsing/__init__.py b/antsibull/docs_parsing/__init__.py index 2dca28074..ded509a57 100644 --- a/antsibull/docs_parsing/__init__.py +++ b/antsibull/docs_parsing/__init__.py @@ -62,3 +62,21 @@ def _get_environment(collection_dir: t.Optional[str]) -> t.Dict[str, str]: if env_var in os.environ: env[env_var] = os.environ[env_var] return env + + +class AnsibleCollectionDocs: + # A nested directory structure that looks like: + # plugin_type: + # plugin_name: # Includes namespace and collection. + # {information from ansible-doc --json. See the ansible-doc documentation for more + # info.} + plugins: t.Dict[str, t.Dict[str, t.Any]] + + # Maps collection name to collection version + collection_versions: t.Dict[str, str] + + def __init__(self, + plugins: t.Dict[str, t.Dict[str, t.Any]], + collection_versions: t.Dict[str, str]): + self.plugins = plugins + self.collection_versions = collection_versions diff --git a/antsibull/docs_parsing/ansible_doc.py b/antsibull/docs_parsing/ansible_doc.py index d4cbad3bd..cb3b19d2a 100644 --- a/antsibull/docs_parsing/ansible_doc.py +++ b/antsibull/docs_parsing/ansible_doc.py @@ -18,7 +18,7 @@ from ..logging import log from ..vendored.json_utils import _filter_non_json_lines from .fqcn import get_fqcn_parts -from . import _get_environment, ParsingError +from . import _get_environment, ParsingError, AnsibleCollectionDocs if t.TYPE_CHECKING: from ..venv import VenvRunner, FakeVenvRunner @@ -158,10 +158,41 @@ async def _get_plugin_info(plugin_type: str, ansible_doc: 'sh.Command', return results +def get_collection_versions(venv: t.Union['VenvRunner', 'FakeVenvRunner'], + collection_dir: t.Optional[str], + collection_names: t.Optional[t.List[str]], + env: t.Dict[str, str], + ) -> t.Dict[str, str]: + collection_versions = {} + + # Obtain ansible.builtin version + if collection_names is None or 'ansible.builtin' in collection_names: + venv_ansible = venv.get_command('ansible') + ansible_version_cmd = venv_ansible('--version', _env=env) + raw_result = ansible_version_cmd.stdout.decode('utf-8', errors='surrogateescape') + for line in raw_result.splitlines(): + if line.startswith('ansible '): + collection_versions['ansible.builtin'] = line[len('ansible '):] + + # Obtain collection versions + venv_ansible_galaxy = venv.get_command('ansible-galaxy') + ansible_collection_list_cmd = venv_ansible_galaxy('collection', 'list', _env=env) + raw_result = ansible_collection_list_cmd.stdout.decode('utf-8', errors='surrogateescape') + for line in raw_result.splitlines(): + parts = line.split() + if len(parts) >= 2: + collection_name = parts[0] + version = parts[1] + if '.' in collection_name: + collection_versions[collection_name] = version + + return collection_versions + + async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], collection_dir: t.Optional[str], collection_names: t.Optional[t.List[str]] = None - ) -> t.Dict[str, t.Dict[str, t.Any]]: + ) -> AnsibleCollectionDocs: """ Retrieve information about all of the Ansible Plugins. @@ -171,12 +202,7 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], search path for Ansible. :arg collection_names: Optional list of collections. If specified, will only collect information for plugins in these collections. - :returns: A nested directory structure that looks like:: - - plugin_type: - plugin_name: # Includes namespace and collection. - {information from ansible-doc --json. See the ansible-doc documentation for more - info.} + :returns: An AnsibleCollectionDocs object. """ flog = mlog.fields(func='get_ansible_plugin_info') flog.debug('Enter') @@ -246,5 +272,8 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], # done so, we want to then fail by raising one of the exceptions. raise ParsingError('Parsing of plugins failed') + flog.debug('Retrieving collection versions') + collection_versions = get_collection_versions(venv, collection_dir, collection_names, env) + flog.debug('Leave') - return plugin_map + return AnsibleCollectionDocs(plugin_map, collection_versions) diff --git a/antsibull/docs_parsing/ansible_internal.py b/antsibull/docs_parsing/ansible_internal.py index 985ad7d19..6bc7a4a9d 100644 --- a/antsibull/docs_parsing/ansible_internal.py +++ b/antsibull/docs_parsing/ansible_internal.py @@ -11,7 +11,7 @@ from ..logging import log from ..utils.get_pkg_data import get_antsibull_data from ..vendored.json_utils import _filter_non_json_lines -from . import _get_environment +from . import _get_environment, AnsibleCollectionDocs if t.TYPE_CHECKING: from ..venv import VenvRunner, FakeVenvRunner @@ -23,7 +23,7 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], collection_dir: t.Optional[str], collection_names: t.Optional[t.List[str]] = None - ) -> t.Dict[str, t.Dict[str, t.Any]]: + ) -> AnsibleCollectionDocs: """ Retrieve information about all of the Ansible Plugins. @@ -33,12 +33,7 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], search path for Ansible. :arg collection_names: Optional list of collections. If specified, will only collect information for plugins in these collections. - :returns: A nested directory structure that looks like:: - - plugin_type: - plugin_name: # Includes namespace and collection. - {information from ansible-doc --json. See the ansible-doc documentation for more - info.} + :returns: An AnsibleCollectionDocs object. """ flog = mlog.fields(func='get_ansible_plugin_info') flog.debug('Enter') @@ -72,7 +67,9 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], plugin_log.fields(error=plugin_data['error']).error( 'Error while extracting documentation. Will not document this plugin.') - # TODO: use result['collections'] + collection_versions = {} + for collection_name, collection_data in result['collections'].items(): + collection_versions[collection_name] = collection_data.get('version') flog.debug('Leave') - return plugin_map + return AnsibleCollectionDocs(plugin_map, collection_versions) diff --git a/antsibull/docs_parsing/parsing.py b/antsibull/docs_parsing/parsing.py index cf066f5b8..daef5de19 100644 --- a/antsibull/docs_parsing/parsing.py +++ b/antsibull/docs_parsing/parsing.py @@ -9,6 +9,7 @@ from ..logging import log from .ansible_doc import get_ansible_plugin_info as ansible_doc_get_ansible_plugin_info from .ansible_internal import get_ansible_plugin_info as ansible_internal_get_ansible_plugin_info +from . import AnsibleCollectionDocs if t.TYPE_CHECKING: from ..venv import VenvRunner, FakeVenvRunner @@ -20,7 +21,7 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], collection_dir: t.Optional[str], collection_names: t.Optional[t.List[str]] = None - ) -> t.Dict[str, t.Dict[str, t.Any]]: + ) -> AnsibleCollectionDocs: """ Retrieve information about all of the Ansible Plugins. @@ -30,12 +31,7 @@ async def get_ansible_plugin_info(venv: t.Union['VenvRunner', 'FakeVenvRunner'], search path for Ansible. :arg collection_names: Optional list of collections. If specified, will only collect information for plugins in these collections. - :returns: A nested directory structure that looks like:: - - plugin_type: - plugin_name: # Includes namespace and collection. - {information from ansible-doc --json. See the ansible-doc documentation for more - info.} + :returns: An AnsibleCollectionDocs object. """ lib_ctx = app_context.lib_ctx.get() diff --git a/antsibull/write_docs.py b/antsibull/write_docs.py index dad162f8b..a86745d79 100644 --- a/antsibull/write_docs.py +++ b/antsibull/write_docs.py @@ -29,7 +29,8 @@ CollectionInfoT = t.Mapping[str, t.Mapping[str, t.Mapping[str, str]]] -async def write_rst(collection_name: str, plugin_short_name: str, plugin_type: str, +async def write_rst(collection_name: str, collection_version: t.Optional[str], + plugin_short_name: str, plugin_type: str, plugin_record: t.Dict[str, t.Any], nonfatal_errors: t.Sequence[str], plugin_tmpl: Template, error_tmpl: Template, dest_dir: str, path_override: t.Optional[str] = None, @@ -38,6 +39,7 @@ async def write_rst(collection_name: str, plugin_short_name: str, plugin_type: s Write the rst page for one plugin. :arg collection_name: Dotted colection name. + :arg collection_version: Collection version (optional). :arg plugin_short_name: short name for the plugin. :arg plugin_type: The type of the plugin. (module, inventory, etc) :arg plugin_record: The record for the plugin. doc, examples, and return are the @@ -68,6 +70,7 @@ async def write_rst(collection_name: str, plugin_short_name: str, plugin_type: s plugin_contents = error_tmpl.render( plugin_type=plugin_type, plugin_name=plugin_name, collection=collection_name, + collection_version=collection_version, nonfatal_errors=nonfatal_errors) else: if nonfatal_errors: @@ -78,6 +81,7 @@ async def write_rst(collection_name: str, plugin_short_name: str, plugin_type: s plugin_name=plugin_name) plugin_contents = plugin_tmpl.render( collection=collection_name, + collection_version=collection_version, plugin_type=plugin_type, plugin_name=plugin_name, doc=plugin_record['doc'], @@ -108,7 +112,8 @@ async def output_all_plugin_rst(collection_info: CollectionInfoT, plugin_info: t.Dict[str, t.Any], nonfatal_errors: PluginErrorsT, dest_dir: str, - squash_hierarchy: bool = False) -> None: + squash_hierarchy: bool = False, + collection_versions: t.Optional[t.Dict[str, str]] = None) -> None: """ Output rst files for each plugin. @@ -121,6 +126,7 @@ async def output_all_plugin_rst(collection_info: CollectionInfoT, :arg squash_hierarchy: If set to ``True``, no directory hierarchy will be used. Undefined behavior if documentation for multiple collections are created. + :arg collection_versions: Optional dictionary mapping collection names to collection versions """ # Setup the jinja environment env = doc_environment(('antsibull.data', 'docsite')) @@ -128,6 +134,9 @@ async def output_all_plugin_rst(collection_info: CollectionInfoT, plugin_tmpl = env.get_template('plugin.rst.j2') error_tmpl = env.get_template('plugin-error.rst.j2') + if collection_versions is None: + collection_versions = {} + writers = [] lib_ctx = app_context.lib_ctx.get() async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool: @@ -136,7 +145,9 @@ async def output_all_plugin_rst(collection_info: CollectionInfoT, for plugin_short_name, dummy_ in plugins.items(): plugin_name = '.'.join((collection_name, plugin_short_name)) writers.append(await pool.spawn( - write_rst(collection_name, plugin_short_name, plugin_type, + write_rst(collection_name, + collection_versions.get(collection_name), + plugin_short_name, plugin_type, plugin_info[plugin_type].get(plugin_name), nonfatal_errors[plugin_type][plugin_name], plugin_tmpl, error_tmpl, dest_dir, squash_hierarchy=squash_hierarchy))) @@ -193,7 +204,8 @@ async def write_plugin_type_index(plugin_type: str, async def write_plugin_lists(collection_name: str, plugin_maps: t.Mapping[str, t.Mapping[str, str]], template: Template, - dest_dir: str) -> None: + dest_dir: str, + collection_version: t.Optional[str]) -> None: """ Write an index page for each collection. @@ -202,10 +214,12 @@ async def write_plugin_lists(collection_name: str, :arg plugin_maps: Mapping of plugin_type to Mapping of plugin_name to short_description. :arg template: A template to render the collection index. :arg dest_dir: The destination directory to output the index into. + :arg collection_version: The collection's version """ index_contents = template.render( collection_name=collection_name, - plugin_maps=plugin_maps) + plugin_maps=plugin_maps, + collection_version=collection_version) # This is only safe because we made sure that the top of the directory tree we're writing to # (docs/docsite/rst) is only writable by us. @@ -289,7 +303,8 @@ async def output_plugin_indexes(collection_info: CollectionInfoT, async def output_indexes(collection_info: CollectionInfoT, dest_dir: str, - squash_hierarchy: bool = False) -> None: + squash_hierarchy: bool = False, + collection_versions: t.Optional[t.Dict[str, str]] = None) -> None: """ Generate collection-level index pages for the collections. @@ -299,10 +314,14 @@ async def output_indexes(collection_info: CollectionInfoT, :arg squash_hierarchy: If set to ``True``, no directory hierarchy will be used. Undefined behavior if documentation for multiple collections are created. + :arg collection_versions: Optional dictionary mapping collection names to collection versions """ flog = mlog.fields(func='output_indexes') flog.debug('Enter') + if collection_versions is None: + collection_versions = {} + env = doc_environment(('antsibull.data', 'docsite')) # Get the templates collection_plugins_tmpl = env.get_template('plugins_by_collection.rst.j2') @@ -328,7 +347,7 @@ async def output_indexes(collection_info: CollectionInfoT, collection_dir = collection_toplevel writers.append(await pool.spawn( write_plugin_lists(collection_name, plugin_maps, collection_plugins_tmpl, - collection_dir))) + collection_dir, collection_versions.get(collection_name)))) await asyncio.gather(*writers)