Resolve wildcards and seen items depth-first; allow wildcards in root…

… nav

mkdocs_literate_nav/parser.py

@@ -3,7 +3,7 @@
 import logging
 import posixpath
 import xml.etree.ElementTree as etree
-from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, Union
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
 
 import markdown
 import markdown.extensions
@@ -20,7 +20,7 @@
 _unescape = markdown.postprocessors.UnescapePostprocessor().run
 
 
-NavItem = Dict[Optional[str], Union[str, Any]]
+NavItem = Union[str, Dict[Optional[str], Union[str, Any]]]
 Nav = List[NavItem]
 RootStack = Tuple[str, ...]
 
@@ -35,43 +35,36 @@ def __init__(
         self.get_nav_for_dir = get_nav_for_dir
         self.globber = globber
         self.implicit_index = implicit_index
+        self.seen_items = set()
 
-    def markdown_to_nav(self, roots: RootStack = ("",)) -> Nav:
-        seen_items = set()
-        if roots.count(roots[0]) > 1:
-            rec = " -> ".join(repr(r) for r in reversed(roots))
-            raise RecursionError(f"Disallowing recursion {rec}")
+    def markdown_to_nav(self, roots: Tuple[str, ...] = (".",)) -> Nav:
         root = roots[0]
         ext = _MarkdownExtension()
         dir_nav = self.get_nav_for_dir(root)
         if dir_nav:
             nav_file_name, md = dir_nav
             markdown.markdown(md, extensions=[ext])
             if ext.nav:
-                seen_items.add(posixpath.normpath(posixpath.join(root, nav_file_name)))
+                self.seen_items.add(posixpath.normpath(posixpath.join(root, nav_file_name)))
         first_item = None
         if ext.nav and self.implicit_index:
             first_item = self.globber.find_index(root)
             if first_item:
-                first_item = _Wildcard(first_item)
+                first_item = Wildcard(root, "/" + first_item, fallback=False)
         if not ext.nav:
             log.debug(f"Navigation for {root!r} will be inferred.")
-            markdown.markdown("* *", extensions=[ext])
-        return self._make_nav(ext.nav, roots, seen_items, first_item)
+            return self._resolve_wildcards([Wildcard(root, "*", fallback=False)], roots)
+        return self._resolve_wildcards(self._list_element_to_nav(ext.nav, root, first_item), roots)
 
-    def _make_nav(
-        self,
-        section: etree.Element,
-        roots: RootStack,
-        seen_items: Set[str],
-        first_item: Optional[str] = None,
-    ) -> Nav:
+    def _list_element_to_nav(
+        self, section: etree.Element, root: str, first_item: Optional[str] = None
+    ):
         assert section.tag in _LIST_TAGS
         result = []
         if first_item is not None:
-            if type(first_item) is str:
-                seen_items.add(first_item)
-            result.append({None: first_item})
+            if isinstance(first_item, str):
+                self.seen_items.add(first_item)
+            result.append(first_item)
         for item in section:
             assert item.tag == "li"
             out_title = item.text
@@ -82,20 +75,11 @@ def _make_nav(
                 child = next(children)
                 if not out_title and child.tag == "a":
                     link = child.get("href")
-                    abs_link = out_item = posixpath.normpath(
-                        posixpath.join(roots[0], link).lstrip("/")
-                    )
-                    if abs_link == ".":
-                        abs_link = ""
-                    if link.endswith("/") and self.globber.isdir(abs_link):
-                        try:
-                            out_item = self.markdown_to_nav((abs_link, *roots))
-                        except RecursionError as e:
-                            log.warning(f"{e} ({link!r})")
+                    out_item = self._maybe_directory_wildcard(root, link)
                     out_title = _unescape("".join(child.itertext()))
                     child = next(children)
                 if child.tag in _LIST_TAGS:
-                    out_item = self._make_nav(child, roots, seen_items, out_item)
+                    out_item = self._list_element_to_nav(child, root, out_item)
                     child = next(children)
             except StopIteration:
                 error = ""
@@ -105,44 +89,103 @@ def _make_nav(
                 error += "Did not find any title specified." + _EXAMPLES
             elif out_item is None:
                 if "*" in out_title:
-                    norm = posixpath.normpath(posixpath.join(roots[0], out_title).lstrip("/"))
-                    if out_title.endswith("/"):
-                        norm += "/"
-                    out_item = _Wildcard(norm)
+                    out_item = Wildcard(root, out_title)
                     out_title = None
                 else:
                     error += "Did not find any item/section content specified." + _EXAMPLES
             if error:
                 raise LiterateNavParseError(error, item)
 
-            if type(out_item) is str:
-                seen_items.add(out_item)
-            result.append({out_title: out_item})
+            if type(out_item) in (str, list, DirectoryWildcard) and out_title is not None:
+                out_item = {out_title: out_item}
+            result.append(out_item)
+        return result
+
+    def _maybe_directory_wildcard(self, root: str, link: str) -> Union["Wildcard", str]:
+        abs_link = posixpath.normpath(posixpath.join(root, link).lstrip("/"))
+        self.seen_items.add(abs_link)
+        if link.endswith("/") and self.globber.isdir(abs_link):
+            return DirectoryWildcard(root, link)
+        return abs_link
+
+    def _resolve_wildcards(self, nav, roots: RootStack = (".",)) -> Nav:
+        def can_recurse(new_root):
+            if new_root in roots:
+                rec = " -> ".join(repr(r) for r in reversed((new_root,) + roots))
+                log.warning(f"Disallowing recursion {rec}")
+                return False
+            return True
+
+        # Ensure depth-first processing, so separate loop for recursive calls first.
+        for entry in nav:
+            if isinstance(entry, dict) and len(entry) == 1:
+                [(key, val)] = entry.items()
+                if isinstance(entry, str):
+                    entry = val
+            if isinstance(entry, str):
+                self.seen_items.add(entry)
+
+        resolved: Nav = []
+        for entry in nav:
+            if isinstance(entry, dict) and len(entry) == 1:
+                [(key, val)] = entry.items()
+                if isinstance(val, list):
+                    entry[key] = self._resolve_wildcards(val, roots)
+                elif isinstance(val, DirectoryWildcard):
+                    entry[key] = (
+                        self.markdown_to_nav((val.value,) + roots)
+                        if can_recurse(val.value)
+                        else val.fallback
+                    )
+                elif isinstance(val, Wildcard):
+                    entry[key] = self._resolve_wildcards([val], roots) or val.fallback
+                if entry[key]:
+                    resolved.append(entry)
+                continue
 
-        # Resolve globs.
-        resolved = []
-        for i, entry in enumerate(result):
-            [(_, top_item)] = entry.items()
-            if not isinstance(top_item, _Wildcard):
+            assert not isinstance(entry, DirectoryWildcard)
+            if not isinstance(entry, Wildcard):
                 resolved.append(entry)
                 continue
-            for item in self.globber.glob(top_item.rstrip("/")):
-                if item in seen_items:
+
+            any_matches = False
+            for item in self.globber.glob(entry.value.rstrip("/")):
+                any_matches = True
+                if item in self.seen_items:
                     continue
                 if self.globber.isdir(item):
                     title = mkdocs.utils.dirname_to_title(posixpath.basename(item))
-                    try:
-                        resolved.append({title: self.markdown_to_nav((item, *roots))})
-                    except RecursionError as e:
-                        log.warning(f"{e} ({item!r})")
-                        resolved.append({title: item})
+                    subitems = self.markdown_to_nav((item,) + roots)
+                    if subitems:
+                        resolved.append({title: subitems})
                 else:
-                    if top_item.endswith("/"):
+                    if entry.value.endswith("/"):
                         continue
                     resolved.append({None: item})
-                seen_items.add(item)
+                self.seen_items.add(item)
+            if not any_matches and entry.fallback:
+                resolved.append(entry.fallback)
         return resolved
 
+    def resolve_yaml_nav(self, nav: Nav) -> Nav:
+        if not isinstance(nav, list):
+            return nav
+        return self._resolve_wildcards([self._resolve_yaml_nav(x) for x in nav])
+
+    def _resolve_yaml_nav(self, item: NavItem):
+        if isinstance(item, str) and "*" in item:
+            return Wildcard("", item)
+        if isinstance(item, dict) and len(item) == 1:
+            [(key, val)] = item.items()
+            if isinstance(val, list):
+                val = [self._resolve_yaml_nav(x) for x in val]
+            elif isinstance(val, str) and "*" in val:
+                val = Wildcard("", val)
+            elif isinstance(val, str):
+                val = self._maybe_directory_wildcard("", val)
+            return {key: val}
+        return item
+
 
 _NAME = "mkdocs_literate_nav"
 
@@ -202,9 +245,22 @@ def run(self, doc):
 """
 
 
-class _Wildcard(str):
-    def __repr__(self):
-        return f"Wildcard({super().__repr__()})"
+class Wildcard:
+    trim_slash = False
+
+    def __init__(self, *path_parts: str, fallback: bool = True):
+        norm = posixpath.normpath(posixpath.join(*path_parts).lstrip("/"))
+        if path_parts[-1].endswith("/") and not self.trim_slash:
+            norm += "/"
+        self.value = norm
+        self.fallback = path_parts[-1] if fallback else None
+
+    def __str__(self):
+        return f"{type(self).__name__}({self.value!r})"
+
+
+class DirectoryWildcard(Wildcard):
+    trim_slash = True
 
 
 def _iter_children_without_tail(element: etree.Element) -> Iterator[etree.Element]:

mkdocs_literate_nav/plugin.py

@@ -3,7 +3,7 @@
 import os
 import os.path
 import posixpath
-from typing import Any, Callable, Iterable, Iterator, Optional, Tuple, Union
+from typing import Iterable, Iterator, Optional, Tuple
 
 import glob2
 import mkdocs.config
@@ -79,37 +79,12 @@ def get_nav_for_dir(path: str) -> Optional[Tuple[str, str]]:
     globber = MkDocsGlobber(files)
     nav_parser = parser.NavParser(get_nav_for_dir, globber, implicit_index=implicit_index)
 
-    def try_resolve_directory(path: str):
-        if path.endswith("/"):
-            path = posixpath.normpath(path.rstrip("/"))
-            if globber.isdir(path):
-                return nav_parser.markdown_to_nav((path,))
-
-    # If nav file is present in the root dir, discard the pre-existing nav.
-    if not nav_data or get_nav_for_dir(""):
-        return try_resolve_directory("/")
-    return convert_strings_in_nav(nav_data, try_resolve_directory)
-
-
-def convert_strings_in_nav(nav_data, converter: Callable[[str], Optional[Any]]):
-    """Walk a nav dict and replace strings in it with the callback."""
-    if isinstance(nav_data, str):
-        new = converter(nav_data)
-        if new is not None:
-            return new
-    elif isinstance(nav_data, dict):
-        return {k: convert_strings_in_nav(v, converter) for k, v in nav_data.items()}
-    elif isinstance(nav_data, list):
-        return list(_flatten(convert_strings_in_nav(v, converter) for v in nav_data))
-    return nav_data
-
-
-def _flatten(items: Iterable[Union[list, Any]]) -> Iterator[Any]:
-    for item in items:
-        if isinstance(item, list):
-            yield from item
-        else:
-            yield item
+    result = None
+    if not nav_data or get_nav_for_dir("."):
+        result = nav_parser.markdown_to_nav()
+    if not result:
+        result = nav_parser.resolve_yaml_nav(nav_data)
+    return result or []
 
 
 class MkDocsGlobber(glob2.Globber):

tests/nav/hybrid/test_empty.yml

@@ -1 +1,2 @@
 files: {}
+output: []

tests/nav/hybrid/test_files_in_subsection.yml

@@ -1,5 +1,5 @@
 nav:
-- foo/:  # Just acts as the title anyway
+- foo/*:  # Just acts as the title anyway
   - Foo: foo/
   - bar/ # Doesn't exist, OK.
 files:
@@ -8,7 +8,7 @@ files:
   foo/zzz.md:
   foo/bar/zzz.md:
 output:
-- foo/:
+- foo/*:
   - Foo:
     - null: foo/xxx.md
     - null: foo/yyy.md

tests/nav/hybrid/test_flattened_files_in_subsection.yml

@@ -1,17 +1,17 @@
 nav:
-- foo/:  # Just acts as the title anyway
-  - foo/
-  - bar/ # Doesn't exist, OK.
+- foo/*:  # Just acts as the title anyway
+  - foo/*
+  - bar/* # Doesn't exist, OK.
 files:
   foo/xxx.md:
   foo/yyy.md:
   foo/zzz.md:
   foo/bar/zzz.md:
 output:
-- foo/:
+- foo/*:
   - null: foo/xxx.md
   - null: foo/yyy.md
   - null: foo/zzz.md
   - Bar:
     - null: foo/bar/zzz.md
-  - bar/
+  - bar/*

tests/nav/hybrid/test_keeps_original_nav.yml

@@ -5,7 +5,7 @@ nav:
   - Test: foo.md
 files: {}
 output:
-- hi-hi: /ho-ho
-- test: test/
+- hi-hi: ho-ho
+- test: test
 - sub:
   - Test: foo.md

tests/nav/hybrid/test_literate_in_subsection.yml

@@ -1,11 +1,12 @@
 nav:
-- foo/:  # Just acts as the title anyway
-  - bar/ # Doesn't exist, OK.
-  - foo/
+- foo/:
+  - Bar: bar/ # Doesn't exist, OK.
+  - Foo: foo/
 files:
   foo/SUMMARY.md: |
     * [Hi](ho.md)
 output:
 - foo/:
-  - bar/
-  - Hi: foo/ho.md
+  - Bar: bar
+  - Foo:
+    - Hi: foo/ho.md

tests/nav/hybrid/test_repeated.yml

@@ -1,11 +1,13 @@
 nav:
-- foo/
-- bar/  # Doesn't exist, OK
-- foo/
+- Foo: foo/
+- Bar: bar/  # Doesn't exist, OK
+- Foo: foo/
 files:
   foo/SUMMARY.md: |
     * [Hi](ho.md)
 output:
-- Hi: foo/ho.md
-- bar/
-- Hi: foo/ho.md
+- Foo:
+  - Hi: foo/ho.md
+- Bar: bar
+- Foo:
+  - Hi: foo/ho.md

tests/nav/hybrid/test_slash_with_nav.yml

@@ -1,5 +1,5 @@
 nav:
-- /
+- '*'
 files:
   SUMMARY.md: |
     * [Hi](ho.md)