[1.7] Add usage docs section (#988) (#1026)

Co-authored-by: Mathieu Martin <mathieu.martin@elastic.co>

CHANGELOG.md

@@ -37,6 +37,7 @@ All notable changes to this project will be documented in this file based on the
 * Introduced `--strict` flag to perform stricter schema validation when running the generator script. #937
 * Added check under `--strict` that ensures composite types in example fields are quoted. #966
 * Added `ignore_above` and `normalizer` support for keyword multi-fields. #971
+* Added ability to supply free-form usage documentation per fieldset. #988
 * Added `--oss` flag for users who want to generate ECS templates for use on OSS clusters. #991
 * Added a new directory with experimental artifacts, which includes all changes
   from RFCs that have reached stage 2. #993

Makefile

@@ -44,7 +44,7 @@ docs:
 	if [ ! -d $(PWD)/build/docs ]; then \
 		git clone --depth=1 https://github.com/elastic/docs.git ./build/docs ; \
 	fi
-	./build/docs/build_docs --asciidoctor --doc ./docs/index.asciidoc --chunk=1 $(OPEN_DOCS) --out ./build/html_docs
+	./build/docs/build_docs --asciidoctor --doc ./docs/index.asciidoc --chunk=2 $(OPEN_DOCS) --out ./build/html_docs
 
 # Alias to generate experimental artifacts
 .PHONY: experimental

docs/usage/README.md

@@ -0,0 +1,40 @@
+# Usage Docs
+
+ECS fields can benefit from additional context and examples which describe their real-world usage. This directory provides a place in the documentation to capture these usage details. AsciiDoc markdown files can be added for any fieldset defined in ECS.
+
+## Adding a Usage Doc
+
+1. Create an AsciiDoc formatted file with the `.asciidoc` file extension.
+2. Save the file in this directory (`docs/usage`), naming it after its associated field set (e.g. a usage document for the fields defined in `schemas/base.yml` fields would be named `docs/usage/base.asciidoc`).
+3. The anchor at the top of the file (e.g. `[[ecs-base-usage]]`) must use the following convention for valid link references in the generated docs: `[[ecs-<<fieldset-name>>-usage]]`.
+4. Run `make`. The asciidoc generator will generate the ECS field reference, including the present usage docs.
+
+If the filename doesn't match a currently defined fieldset, the usage document will not appear on the ECS docs site. This logic is handled in the AsciiDoc generator scripts, `scripts/generators/asciidoc_fields.py`.
+
+## Template
+
+The following is a simple AsciiDoc template as a starting point:
+
+```asciidoc
+
+[[ecs-fieldset-usage]]
+==== Fieldset Usage
+
+Add relevant text here.
+
+[discrete]
+===== New Section header
+
+Text for the new section.
+
+[discrete]
+===== Examples
+
+[source,sh]
+-----------
+{
+    "key": "value"
+}
+-----------
+
+```

scripts/generators/asciidoc_fields.py

@@ -72,6 +72,15 @@ def sort_fields(fieldset):
     return sorted(fields_list, key=lambda field: field['name'])
 
 
+def check_for_usage_doc(fieldset_name, usage_file_list=ecs_helpers.usage_doc_files()):
+    """Checks if a usage doc exists for the specified
+       fieldset.
+
+    :param fieldset_name: The name of the target fieldset
+    """
+    return f"{fieldset_name}.asciidoc" in usage_file_list
+
+
 def templated(template_name):
     """Decorator function to simplify rendering a template.
 
@@ -138,10 +147,12 @@ def generate_field_details_page(fieldset):
     sorted_reuse_fields = render_fieldset_reuse_text(fieldset)
     render_nestings_reuse_fields = render_nestings_reuse_section(fieldset)
     sorted_fields = sort_fields(fieldset)
+    usage_doc = check_for_usage_doc(fieldset.get('name'))
     return dict(fieldset=fieldset,
                 sorted_reuse_fields=sorted_reuse_fields,
                 render_nestings_reuse_section=render_nestings_reuse_fields,
-                sorted_fields=sorted_fields)
+                sorted_fields=sorted_fields,
+                usage_doc=usage_doc)
 
 
 # Allowed values section

scripts/generators/ecs_helpers.py

@@ -2,6 +2,7 @@
 import os
 import yaml
 import git
+import pathlib
 import warnings
 
 from collections import OrderedDict
@@ -113,6 +114,14 @@ def get_tree_by_ref(ref):
     return commit.tree
 
 
+def usage_doc_files():
+    usage_docs_dir = os.path.join(os.path.dirname(__file__), '../../docs/usage')
+    usage_docs_path = pathlib.Path(usage_docs_dir)
+    if usage_docs_path.is_dir():
+        return [x.name for x in usage_docs_path.glob('*.asciidoc') if x.is_file()]
+    return []
+
+
 def ecs_files():
     """Return the schema file list to load"""
     schema_glob = os.path.join(os.path.dirname(__file__), '../../schemas/*.yml')

scripts/templates/field_details.j2

@@ -4,6 +4,12 @@
 
 {{ fieldset['description']|replace("\n", "\n\n") }}
 
+{%- if usage_doc %}
+
+Find additional usage and examples in the {{ fieldset['name'] }} fields <<ecs-{{ fieldset['name']|replace("_", "-") }}-usage,usage>> section.
+
+{% endif %}
+
 {# Field Details Table Header -#}
 [discrete]
 ==== {{ fieldset['title'] }} Field Details
@@ -113,4 +119,9 @@ Note also that the `{{ fieldset['name'] }}` fields are not expected to be used d
 |=====
 
 {% endif %}{# if 'nestings' #}
-{%- endif %}{# if 'nestings' or 'reusable' in fieldset #}
+{%- endif -%}{# if 'nestings' or 'reusable' in fieldset #}
+{%- if usage_doc %}
+
+include::usage/{{ fieldset['name'] }}.asciidoc[]
+
+{% endif %}

scripts/tests/test_asciidoc_fields.py

@@ -127,6 +127,16 @@ def test_rendering_fieldset_nesting(self):
         self.assertEqual('as', foo_nesting_fields[0]['name'])
         self.assertEqual('Fields describing an AS', foo_nesting_fields[0]['short'])
 
+    def test_check_for_usage_doc_true(self):
+        usage_files = ["foo.asciidoc"]
+        foo_name = self.foo_fieldset.get('name')
+        self.assertTrue(asciidoc_fields.check_for_usage_doc(foo_name, usage_file_list=usage_files))
+
+    def test_check_for_usage_doc_false(self):
+        usage_files = ["notfoo.asciidoc"]
+        foo_name = self.foo_fieldset.get('name')
+        self.assertFalse(asciidoc_fields.check_for_usage_doc(foo_name, usage_file_list=usage_files))
+
 
 if __name__ == '__main__':
     unittest.main()