From 93a53bfba073538bce78ed9a23d2526761d6f435 Mon Sep 17 00:00:00 2001 From: Mark Gillard Date: Tue, 4 Oct 2022 13:52:49 +1100 Subject: [PATCH] navbar overhaul - fixed custom navbar items always being transformed to lowercase - fixed navbar generating links to empty pages - fixed `navbar` option allowing duplicates - added `navbar` values `all` and `default` - added `concepts` to the default set of links in `navbar` also: - removed command-line option `dry` - reduced I/O churn during HTML post-processing - minor style fixes - minor fixes + refactors --- CHANGELOG.md | 11 +- README.md | 18 +- poxy/data/css/poxy-overrides.css | 12 +- poxy/data/generated/poxy.css | 10 +- .../templates/doxygen/namespace.html | 2 +- poxy/fixers.py | 70 +- poxy/main.py | 46 +- poxy/project.py | 155 +- poxy/run.py | 1949 +++++++++-------- poxy/soup.py | 25 +- poxy/svg.py | 2 +- poxy/utils.py | 4 +- tests/regenerate_tests.py | 28 +- .../expected_html/annotated.html | 24 +- .../expected_html/concepts.html | 24 +- .../expected_html/files.html | 24 +- .../expected_html/index.html | 24 +- .../expected_html/modules.html | 24 +- .../expected_html/namespaces.html | 24 +- .../expected_html/pages.html | 24 +- .../test_empty_project/expected_xml/index.xml | 2 +- .../expected_html/Test!.tagfile.xml | 54 - .../test_project/expected_html/annotated.html | 24 +- ...1empty.html => classtest_1_1class__1.html} | 24 +- ...l => classtest_1_1template__class__1.html} | 22 +- tests/test_project/expected_html/code_8h.html | 40 +- .../test_project/expected_html/concepts.html | 30 +- ...t2.html => concepttest_1_1concept__1.html} | 20 +- ... concepttest_1_1nested_1_1concept__2.html} | 20 +- .../dir_68267d1309a1af8e8297ef4c3efbcdba.html | 16 +- tests/test_project/expected_html/files.html | 16 +- tests/test_project/expected_html/index.html | 16 +- tests/test_project/expected_html/modules.html | 16 +- .../expected_html/namespaces.html | 22 +- ...namespacetest1.html => namespacetest.html} | 41 +- ...cept1.html => namespacetest_1_1empty.html} | 36 +- ...est2.html => namespacetest_1_1nested.html} | 24 +- tests/test_project/expected_html/pages.html | 16 +- .../expected_html/searchdata-v2.js | 2 +- ...1foo.html => structtest_1_1struct__1.html} | 32 +- ...cttest_1_1struct__1_1_1nested__struct.html | 118 + .../expected_xml/classtest_1_1class__1.xml | 16 + ...ml => classtest_1_1template__class__1.xml} | 10 +- tests/test_project/expected_xml/code_8h.xml | 14 +- .../expected_xml/concepttest1_1_1concept2.xml | 34 - .../concepttest1_1_1test2_1_1concept3.xml | 34 - ...ept1.xml => concepttest_1_1concept__1.xml} | 10 +- .../concepttest_1_1nested_1_1concept__2.xml | 34 + .../dir_68267d1309a1af8e8297ef4c3efbcdba.xml | 2 +- tests/test_project/expected_xml/index.xml | 20 +- .../expected_xml/namespacetest.xml | 18 + .../expected_xml/namespacetest1.xml | 16 - .../expected_xml/namespacetest1_1_1test2.xml | 12 - ..._1empty.xml => namespacetest_1_1empty.xml} | 10 +- .../expected_xml/namespacetest_1_1nested.xml | 12 + .../expected_xml/structtest1_1_1foo.xml | 17 - .../expected_xml/structtest_1_1struct__1.xml | 17 + ...ucttest_1_1struct__1_1_1nested__struct.xml | 16 + tests/test_project/poxy.toml | 2 +- tests/test_project/src/code.h | 62 +- theme_test/index.html | 2 +- 61 files changed, 1639 insertions(+), 1810 deletions(-) delete mode 100644 tests/test_project/expected_html/Test!.tagfile.xml rename tests/test_project/expected_html/{namespacetest1_1_1empty.html => classtest_1_1class__1.html} (70%) rename tests/test_project/expected_html/{classtest1_1_1foo_1_1bar.html => classtest_1_1template__class__1.html} (72%) rename tests/test_project/expected_html/{concepttest1_1_1concept2.html => concepttest_1_1concept__1.html} (73%) rename tests/test_project/expected_html/{concepttest1_1_1test2_1_1concept3.html => concepttest_1_1nested_1_1concept__2.html} (72%) rename tests/test_project/expected_html/{namespacetest1.html => namespacetest.html} (69%) rename tests/test_project/expected_html/{concepttest1_1_1foo_1_1concept1.html => namespacetest_1_1empty.html} (65%) rename tests/test_project/expected_html/{namespacetest1_1_1test2.html => namespacetest_1_1nested.html} (71%) rename tests/test_project/expected_html/{structtest1_1_1foo.html => structtest_1_1struct__1.html} (69%) create mode 100644 tests/test_project/expected_html/structtest_1_1struct__1_1_1nested__struct.html create mode 100644 tests/test_project/expected_xml/classtest_1_1class__1.xml rename tests/test_project/expected_xml/{classtest1_1_1foo_1_1bar.xml => classtest_1_1template__class__1.xml} (67%) delete mode 100644 tests/test_project/expected_xml/concepttest1_1_1concept2.xml delete mode 100644 tests/test_project/expected_xml/concepttest1_1_1test2_1_1concept3.xml rename tests/test_project/expected_xml/{concepttest1_1_1foo_1_1concept1.xml => concepttest_1_1concept__1.xml} (75%) create mode 100644 tests/test_project/expected_xml/concepttest_1_1nested_1_1concept__2.xml create mode 100644 tests/test_project/expected_xml/namespacetest.xml delete mode 100644 tests/test_project/expected_xml/namespacetest1.xml delete mode 100644 tests/test_project/expected_xml/namespacetest1_1_1test2.xml rename tests/test_project/expected_xml/{namespacetest1_1_1empty.xml => namespacetest_1_1empty.xml} (50%) create mode 100644 tests/test_project/expected_xml/namespacetest_1_1nested.xml delete mode 100644 tests/test_project/expected_xml/structtest1_1_1foo.xml create mode 100644 tests/test_project/expected_xml/structtest_1_1struct__1.xml create mode 100644 tests/test_project/expected_xml/structtest_1_1struct__1_1_1nested__struct.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 11f624a..e31d370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,16 @@ # Changelog -## Unreleased +## v0.9.1 - 2022-10-04 - fixed SVG inlining not preserving original image class attributes -- fixed ValueError when reading some SVG files +- fixed `ValueError` when reading some SVG files +- fixed `navbar` option allowing duplicates +- fixed custom navbar items always being transformed to lowercase +- fixed navbar generating links to empty pages +- added `concepts` to the default set of links in `navbar` +- added `navbar` values `all` and `default` +- reduced I/O churn during HTML post-processing +- removed command-line option `dry` ## v0.9.0 - 2022-10-03 diff --git a/README.md b/README.md index ce3093d..fa77a07 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,9 @@ pip install poxy Poxy is a command-line application. ``` -poxy [-h] [-v] [--doxygen ] [--dry] [--threads N] [--version] [--werror] [--xmlonly] - [--ppinclude ] [--ppexclude ] [--theme {auto,light,dark,custom}] - [config] +poxy [-h] [-v] [--doxygen ] [--ppinclude ] [--ppexclude ] + [--theme {auto,light,dark,custom}] [--threads N] [--version] [--werror] [--xmlonly] + [config] Generate fancy C++ documentation. @@ -79,15 +79,14 @@ options: -h, --help show this help message and exit -v, --verbose enable very noisy diagnostic output --doxygen specify the Doxygen executable to use (default: find on system path) - --dry do a 'dry run' only, stopping after emitting the effective Doxyfile - --threads N set the number of threads to use (default: automatic) - --version print the version and exit - --werror always treat warnings as errors regardless of config file settings - --xmlonly stop after generating and preprocessing the Doxygen xml --ppinclude pattern matching HTML file names to post-process (default: all) --ppexclude pattern matching HTML file names to exclude from post-processing (default: none) --theme {auto,light,dark,custom} the CSS theme to use (default: auto) + --threads N set the number of threads to use (default: automatic) + --version print the version and exit + --werror always treat warnings as errors regardless of config file settings + --xmlonly stop after generating and preprocessing the Doxygen xml ``` The basic three-step to using Poxy is similar to Doxygen: @@ -96,9 +95,6 @@ The basic three-step to using Poxy is similar to Doxygen: 2. Invoke Poxy on it: `poxy path/to/poxy.toml` (or simply `poxy` if the cwd contains the config file) 3. See your HTML documentation `/html` -ℹ️ If there exists a `Doxyfile` or `Doxyfile-mcss` in the same directory as your `poxy.toml` it will be loaded -first, then the Poxy overrides applied on top of it. Otherwise a 'default' Doxyfile is used as the base. -

## Config file options diff --git a/poxy/data/css/poxy-overrides.css b/poxy/data/css/poxy-overrides.css index 70fc699..8f01c3b 100644 --- a/poxy/data/css/poxy-overrides.css +++ b/poxy/data/css/poxy-overrides.css @@ -150,8 +150,8 @@ body > header > nav a:hover margin-top: 1.0rem; } -/* github badges (index.html) */ -.gh-badges +/* index page badges */ +#poxy-badges { padding-bottom: 0.25rem; margin-left: -0.9rem; @@ -160,24 +160,24 @@ body > header > nav a:hover } @media screen and (min-width: 992px) { - .gh-badges + #poxy-badges { display: flex; justify-content: space-between; } - .gh-badges br + #poxy-badges br { display: none; } } @media screen and (max-width: 992px) { - .gh-badges + #poxy-badges { text-align: center; line-height: 1.75rem; } - .gh-badges a + #poxy-badges a { margin-left: 0.2rem; margin-right: 0.2rem; diff --git a/poxy/data/generated/poxy.css b/poxy/data/generated/poxy.css index d519233..da7627f 100644 --- a/poxy/data/generated/poxy.css +++ b/poxy/data/generated/poxy.css @@ -2667,14 +2667,14 @@ body > header > nav a:hover { text-decoration: none !important; } .m-doc-details div > table.m-table td > p > strong > em { display: block; } .m-doc-details div > table.m-table td > strong:not(:first-child) > em, .m-doc-details div > table.m-table td > p:not(:first-child) > strong > em { margin-top: 1.0rem; } -.gh-badges { padding-bottom: 0.25rem; margin-left: -0.9rem; margin-right: -0.9rem; margin-top: -0.5rem; } +#poxy-badges { padding-bottom: 0.25rem; margin-left: -0.9rem; margin-right: -0.9rem; margin-top: -0.5rem; } @media screen and (min-width: 992px) { -.gh-badges { display: flex; justify-content: space-between; } -.gh-badges br { display: none; } +#poxy-badges { display: flex; justify-content: space-between; } +#poxy-badges br { display: none; } } @media screen and (max-width: 992px) { -.gh-badges { text-align: center; line-height: 1.75rem; } -.gh-badges a { margin-left: 0.2rem; margin-right: 0.2rem; margin-bottom: 0.4rem; } +#poxy-badges { text-align: center; line-height: 1.75rem; } +#poxy-badges a { margin-left: 0.2rem; margin-right: 0.2rem; margin-bottom: 0.4rem; } } h1 span.m-thin { color: var(--dim-color); } #poxy-main-banner { diff --git a/poxy/data/m.css/documentation/templates/doxygen/namespace.html b/poxy/data/m.css/documentation/templates/doxygen/namespace.html index 267d439..e79fb82 100644 --- a/poxy/data/m.css/documentation/templates/doxygen/namespace.html +++ b/poxy/data/m.css/documentation/templates/doxygen/namespace.html @@ -8,7 +8,7 @@

{# need an explicit space here otherwise the newline gets removed #} {% if compound.include %} -
#include {{ compound.include[0] }}
+
#include {{ compound.include[0] }}
{% endif %}

{% endblock %} diff --git a/poxy/fixers.py b/poxy/fixers.py index 3658af4..90d68e3 100644 --- a/poxy/fixers.py +++ b/poxy/fixers.py @@ -11,6 +11,7 @@ from bs4 import NavigableString from .utils import * from .svg import SVG +from .project import Context from . import soup #======================================================================================================================= @@ -107,7 +108,7 @@ def __single_tags_substitute(cls, m, out, context): else: return f'<{m[1]}{(" " + tag_content) if tag_content else ""}>' - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): if doc.article_content is None: return False changed = False @@ -195,7 +196,7 @@ class CPPModifiers1(_CPPModifiersBase): def __substitute(cls, m, out): return f'{m[1]}{m[2]}{m[3]}' - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): if doc.article_content is None: return False changed = False @@ -221,7 +222,7 @@ def __substitute(cls, m, matches): matches.append(m[1]) return ' ' - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): if doc.article_content is None: return False changed = False @@ -267,7 +268,7 @@ class CPPTemplateTemplate(HTMLFixer): def __substitute(cls, m): return f'{m[1]}
\n{m[2]}' - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): changed = False for template in doc.body('div', class_='m-doc-template'): replacer = RegexReplacer(self.__expression, lambda m, out: self.__substitute(m), str(template)) @@ -283,7 +284,7 @@ class StripIncludes(HTMLFixer): Strips #include based on context.sources.strip_includes. ''' - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): if doc.article is None or not context.sources.strip_includes: return False changed = False @@ -314,8 +315,8 @@ class Banner(HTMLFixer): Makes the first image on index.html a 'banner' ''' - def __call__(self, doc, context): - if doc.article_content is None or doc.path.name.lower() != 'index.html': + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): + if doc.article_content is None or path.name.lower() != 'index.html': return False parent = doc.article_content @@ -333,15 +334,17 @@ def __call__(self, doc, context): banner = banner.extract() h1.replace_with(banner) banner[r'id'] = r'poxy-main-banner' + soup.add_class(doc.body, r'poxy-has-main-banner') if context.badges: - parent = doc.new_tag('div', class_='gh-badges', after=banner) + parent = doc.new_tag('div', id='poxy-badges', after=banner) for (alt, src, href) in context.badges: if alt is None and src is None and href is None: doc.new_tag('br', parent=parent) else: anchor = doc.new_tag('a', parent=parent, href=href, target='_blank') doc.new_tag('img', parent=anchor, src=src, alt=alt) + soup.add_class(doc.body, r'poxy-has-badges') return True @@ -447,7 +450,7 @@ def __adjacent_maybe_by_whitespace(cls, a, b): return False return True - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): changed = False # fix up syntax highlighting @@ -626,7 +629,7 @@ def __substitute(cls, m, uri): external = uri.startswith('http') return rf'''{m[0]}''' - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): if doc.article_content is None: return False @@ -658,7 +661,7 @@ def m_doc_anchor_tags(tag): continue # don't override internal self-links if anchor != -1: href = href[:anchor] - if href == uri or href == doc.path.name: # don't override internal self-links + if href == uri or href == path.name: # don't override internal self-links continue link['href'] = uri soup.set_class(link, ['m-doc', 'poxy-injected']) @@ -681,7 +684,7 @@ def m_doc_anchor_tags(tag): strings = strings + soup.string_descendants(tag, lambda t: soup.find_parent(t, 'a', tag) is None) strings = [s for s in strings if s.parent is not None] for expr, uri in context.autolinks: - if uri == doc.path.name: # don't create unnecessary self-links + if uri == path.name: # don't create unnecessary self-links continue i = 0 while i < len(strings): @@ -717,7 +720,7 @@ class Links(HTMLFixer): __godbolt = re.compile(r'^\s*https[:]//godbolt.org/z/.+?$', re.I) __local_href = re.compile(r'^([-/_a-zA-Z0-9]+\.[a-zA-Z]+)(?:#(.*))?$') - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): changed = False for anchor in doc.body('a', href=True): href = anchor['href'] @@ -746,7 +749,7 @@ def __call__(self, doc, context): # make sure links to local files point to actual existing files match = self.__local_href.fullmatch(href) - if match and not coerce_path(doc.path.parent, match[1]).exists(): + if match and not coerce_path(path.parent, match[1]).exists(): changed = True if is_mdoc: href = r'#' @@ -787,7 +790,7 @@ class EmptyTags(HTMLFixer): Prunes the tree of various empty tags (happens as a side-effect of some other operations). ''' - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): changed = False for tag in doc.body((r'p', r'span')): if not tag.contents or ( @@ -804,7 +807,7 @@ class MarkTOC(HTMLFixer): Marks any table-of-contents with a custom class. ''' - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): if doc.table_of_contents is None: return False soup.add_class(doc.table_of_contents, r'poxy-toc') @@ -819,7 +822,7 @@ class InjectSVGs(HTMLFixer): Injects the contents of SVG tags directly into the document. ''' - def __call__(self, doc, context): + def __call__(self, context: Context, doc: soup.HTMLDocument, path: Path): imgs = doc.body.find_all(r'img') if not imgs: return False @@ -829,7 +832,7 @@ def __call__(self, doc, context): ] count = 0 for img in imgs: - src = Path(doc.path.parent, img[r'src']) + src = Path(path.parent, img[r'src']) if not src.exists() or not src.is_file() or src.stat().st_size > (1024 * 16): # max 16 kb continue svg = SVG( @@ -856,15 +859,13 @@ class ImplementationDetails(PlainTextFixer): ''' __shorthands = ((r'POXY_IMPLEMENTATION_DETAIL_IMPL', r'/* ... */'), ) - def __call__(self, doc, context): - changed = False + def __call__(self, context: Context, text: str, path: Path) -> str: for shorthand, replacement in self.__shorthands: - idx = doc[0].find(shorthand) + idx = text.find(shorthand) while idx >= 0: - doc[0] = doc[0][:idx] + replacement + doc[0][idx + len(shorthand):] - changed = True - idx = doc[0].find(shorthand) - return changed + text = text[:idx] + replacement + text[idx + len(shorthand):] + idx = text.find(shorthand) + return text @@ -873,14 +874,11 @@ class MarkdownPages(PlainTextFixer): Cleans up some HTML snafus from markdown-based pages. ''' - def __call__(self, doc, context): - if not doc[1].name.lower().startswith(r'md_') and not doc[1].name.lower().startswith(r'm_d__'): - return False - - WBR = r'(?:)?' - PREFIX = rf'_{WBR}_{WBR}poxy_{WBR}thiswasan_{WBR}' - doc[0] = re.sub(rf'{PREFIX}amp', r'&', doc[0]) - doc[0] = re.sub(rf'{PREFIX}at', r'@', doc[0]) - doc[0] = re.sub(rf'{PREFIX}fe0f', r'️', doc[0]) - - return True + def __call__(self, context: Context, text: str, path: Path) -> str: + if path.name.lower().startswith(r'md_') or path.name.lower().startswith(r'm_d__'): + WBR = r'(?:)?' + PREFIX = rf'_{WBR}_{WBR}poxy_{WBR}thiswasan_{WBR}' + text = re.sub(rf'{PREFIX}amp', r'&', text) + text = re.sub(rf'{PREFIX}at', r'@', text) + text = re.sub(rf'{PREFIX}fe0f', r'️', text) + return text diff --git a/poxy/main.py b/poxy/main.py index 6b4e65a..65ca77a 100644 --- a/poxy/main.py +++ b/poxy/main.py @@ -56,7 +56,7 @@ def main(invoker=True): r'config', type=Path, nargs='?', - default=None, + default=Path('.'), help=r'path to poxy.toml or a directory containing it (default: %(default)s)' ) args.add_argument( @@ -73,9 +73,24 @@ def main(invoker=True): help=r"specify the Doxygen executable to use (default: find on system path)" ) args.add_argument( - r'--dry', # - action=r'store_true', - help=r"do a 'dry run' only, stopping after emitting the effective Doxyfile" + r'--ppinclude', # + type=str, + default=None, + metavar=r'', + help=r"pattern matching HTML file names to post-process (default: all)" + ) + args.add_argument( + r'--ppexclude', # + type=str, + default=None, + metavar=r'', + help=r"pattern matching HTML file names to exclude from post-processing (default: none)" + ) + args.add_argument( + r'--theme', # + choices=[r'auto', r'light', r'dark', r'custom'], + default=r'auto', + help=r'the CSS theme to use (default: %(default)s)' ) args.add_argument( r'--threads', # @@ -100,26 +115,6 @@ def main(invoker=True): action=r'store_true', help=r"stop after generating and preprocessing the Doxygen xml" ) - args.add_argument( - r'--ppinclude', # - type=str, - default=None, - metavar=r'', - help=r"pattern matching HTML file names to post-process (default: all)" - ) - args.add_argument( - r'--ppexclude', # - type=str, - default=None, - metavar=r'', - help=r"pattern matching HTML file names to exclude from post-processing (default: none)" - ) - args.add_argument( - r'--theme', # - choices=[r'auto', r'light', r'dark', r'custom'], - default=r'auto', - help=r'the CSS theme to use (default: %(default)s)' - ) #-------------------------------------------------------------- # hidden developer-only/diagnostic arguments #-------------------------------------------------------------- @@ -223,7 +218,7 @@ def main(invoker=True): if args.update_styles or args.update_fonts or args.update_emoji or args.mcss is not None: return - with ScopeTimer(r'All tasks', print_start=False, print_end=not args.dry) as timer: + with ScopeTimer(r'All tasks', print_start=False, print_end=True) as timer: run( config_path=args.config, output_dir=Path.cwd(), @@ -232,7 +227,6 @@ def main(invoker=True): verbose=args.verbose, doxygen_path=args.doxygen, logger=True, # stderr + stdout - dry_run=args.dry, xml_only=args.xmlonly, html_include=args.ppinclude, html_exclude=args.ppexclude, diff --git a/poxy/project.py b/poxy/project.py index 89197e1..bcdb35c 100644 --- a/poxy/project.py +++ b/poxy/project.py @@ -659,7 +659,8 @@ class Defaults(object): r'(?:::)?TArray(?:s)?': r'https://docs.unrealengine.com/4.27/en-US/API/Runtime/Core/Containers/TArray/', } - navbar = [r'files', r'groups', r'namespaces', r'classes'] + navbar = (r'files', r'groups', r'namespaces', r'classes', r'concepts') + navbar_all = (r'pages', *navbar, r'repo', r'theme') aliases = { # poxy r'cpp': @@ -1264,13 +1265,12 @@ def verbose_object(self, name, obj): self.verbose_value(rf'{name}.{k}', v) def __init__( - self, config_path, output_dir, threads, cleanup, verbose, doxygen_path, logger, dry_run, xml_only, html_include, + self, config_path, output_dir, threads, cleanup, verbose, doxygen_path, logger, xml_only, html_include, html_exclude, treat_warnings_as_errors, theme, copy_assets ): self.logger = logger self.__verbose = bool(verbose) - self.dry_run = bool(dry_run) self.xml_only = bool(xml_only) self.cleanup = bool(cleanup) self.copy_assets = bool(copy_assets) @@ -1278,14 +1278,12 @@ def __init__( self.version = lib_version() self.version_string = r'.'.join([str(v) for v in lib_version()]) - if not self.dry_run or self.__verbose: - self.info(rf'Poxy v{self.version_string}') + self.info(rf'Poxy v{self.version_string}') - self.verbose_value(r'Context.dry_run', self.dry_run) self.verbose_value(r'Context.xml_only', self.xml_only) self.verbose_value(r'Context.cleanup', self.cleanup) - threads = int(threads) + threads = int(threads) if threads is not None else 0 if threads <= 0: threads = os.cpu_count() self.threads = max(1, min(os.cpu_count(), threads)) @@ -1380,13 +1378,12 @@ def __init__( assert self.temp_pages_dir.is_absolute() # delete leftovers from previous run and initialize various dirs - if not self.dry_run: - delete_directory(self.xml_dir, logger=self.verbose_logger) - delete_directory(self.temp_dir, logger=self.verbose_logger) - if not self.xml_only: - delete_directory(self.html_dir, logger=self.verbose_logger) - self.temp_dir.mkdir(exist_ok=True, parents=True) - self.temp_pages_dir.mkdir(exist_ok=True, parents=True) + delete_directory(self.xml_dir, logger=self.verbose_logger) + delete_directory(self.temp_dir, logger=self.verbose_logger) + if not self.xml_only: + delete_directory(self.html_dir, logger=self.verbose_logger) + self.temp_dir.mkdir(exist_ok=True, parents=True) + self.temp_pages_dir.mkdir(exist_ok=True, parents=True) # doxygen if doxygen_path is not None: @@ -1458,9 +1455,7 @@ def add_internal_asset(p): config = dict() if self.config_path.exists(): assert_existing_file(self.config_path) - config = pytomlpp.loads( - read_all_text_from_file(self.config_path, logger=self.verbose_logger if dry_run else self.logger) - ) + config = pytomlpp.loads(read_all_text_from_file(self.config_path, logger=self.logger)) config = assert_no_unexpected_keys(config, self.__config_schema.validate(config)) self.warnings = Warnings(config) @@ -1651,7 +1646,7 @@ def add_internal_asset(p): if self.changelog or candidate_dir.parent == candidate_dir: break candidate_dir = candidate_dir.parent - if not self.changelog and not self.dry_run: + if not self.changelog: self.warning( rf'changelog: Option was set to true but no file with a known changelog file name could be found! Consider using an explicit path.' ) @@ -1662,11 +1657,11 @@ def add_internal_asset(p): self.changelog = Path(self.input_dir, self.changelog) if not self.changelog.exists() or not self.changelog.is_file(): raise Error(rf'changelog: {config["changelog"]} did not exist or was not a file') + if self.changelog: + temp_changelog_path = Path(self.temp_pages_dir, r'poxy_changelog.md') + copy_file(self.changelog, temp_changelog_path, logger=self.verbose_logger) + self.changelog = temp_changelog_path self.verbose_value(r'Context.changelog', self.changelog) - if self.changelog and not self.dry_run: - self.temp_pages_dir.mkdir(exist_ok=True, parents=True) - copy_file(self.changelog, Path(self.temp_pages_dir, r'poxy_changelog.md'), logger=self.verbose_logger) - self.changelog = Path(self.temp_pages_dir, r'poxy_changelog.md') # sources (INPUT, FILE_PATTERNS, STRIP_FROM_PATH, STRIP_FROM_INC_PATH, EXTRACT_ALL) self.sources = Sources( @@ -1674,10 +1669,11 @@ def add_internal_asset(p): r'sources', self.input_dir, additional_inputs=( - self.temp_pages_dir if not self.dry_run else None, - self.changelog if self.changelog and not self.dry_run else None, *[f for f, d in self.blog_files] + self.temp_pages_dir, # + self.changelog if self.changelog else None, + *[f for f, d in self.blog_files] ), - additional_strip_paths=(self.temp_pages_dir if not self.dry_run else None, ) + additional_strip_paths=(self.temp_pages_dir, ) ) self.verbose_object(r'Context.sources', self.sources) @@ -1723,45 +1719,76 @@ def add_internal_asset(p): assert_existing_file(k) self.verbose_value(r'Context.tagfiles', self.tagfiles) - # m.css navbar - if r'navbar' in config: + # navbar + if 1: + # initialize self.navbar = [] - for v in coerce_collection(config['navbar']): - val = v.strip().lower() - if val: - self.navbar.append(val) - else: - self.navbar = copy.deepcopy(Defaults.navbar) - for i in range(len(self.navbar)): - if self.navbar[i] == 'classes': - self.navbar[i] = 'annotated' - elif self.navbar[i] == 'groups': - self.navbar[i] = 'modules' - # repo buttons - if not self.repo: - # remove all repo buttons if there's no repo - for KEY in (r'repo', r'repository', *repos.KEYS): - if KEY in self.navbar: - self.navbar.remove(KEY) - else: - # remove repo buttons matching an uninstantiated repo type - for TYPE in repos.TYPES: - if not isinstance(self.repo, TYPE) and TYPE.KEY in self.navbar: - self.navbar.remove(TYPE.KEY) - # sub all remaining repo key aliases for simply 'repo' + if r'navbar' in config: + for v in coerce_collection(config['navbar']): + val = v.strip() + if val: + self.navbar.append(val) + else: + self.navbar = list(copy.deepcopy(Defaults.navbar)) + + # expand 'default' and 'all' + new_navbar = [] + for link in self.navbar: + if link == r'all': + new_navbar += [*Defaults.navbar_all] + elif link == r'default': + new_navbar += [*Defaults.navbar] + else: + new_navbar.append(link) + self.navbar = new_navbar + + # normalize aliases for i in range(len(self.navbar)): - if self.navbar[i] in (r'repository', *repos.KEYS): + if self.navbar[i] == r'annotated': # 'annotated' is doxygen-speak for 'classes' + self.navbar[i] = r'classes' + elif self.navbar[i] == r'modules': # 'modules' is doxygen-speak for 'groups' + self.navbar[i] = r'groups' + elif self.navbar[i] == r'repository': self.navbar[i] = r'repo' - # add a repo button to the end if none was present - if r'repo' not in self.navbar: - self.navbar.append(r'repo') - # theme button - if self.theme != r'custom' and r'theme' not in self.navbar: - self.navbar.append(r'theme') - if self.theme == r'custom' and r'theme' in self.navbar: - self.navbar.remove(r'theme') - self.navbar = tuple(self.navbar) - self.verbose_value(r'Context.navbar', self.navbar) + + # repo logic + if not self.repo: + for KEY in (r'repo', *repos.KEYS): + if KEY in self.navbar: + self.navbar.remove(KEY) + else: + # remove repo buttons matching an uninstantiated repo type + for TYPE in repos.TYPES: + if not isinstance(self.repo, TYPE) and TYPE.KEY in self.navbar: + self.navbar.remove(TYPE.KEY) + # sub all remaining repo key aliases for simply 'repo' + for i in range(len(self.navbar)): + if self.navbar[i] in repos.KEYS: + self.navbar[i] = r'repo' + # add a repo button to the end if none was present + if r'repo' not in self.navbar: + self.navbar.append(r'repo') + + # theme logic + if self.theme != r'custom' and r'theme' not in self.navbar: + self.navbar.append(r'theme') + if self.theme == r'custom' and r'theme' in self.navbar: + self.navbar.remove(r'theme') + + # remove duplicate keywords + seen_keywords = set() + new_navbar = [] + for nav in self.navbar: + if nav in ( + r'files', r'pages', r'modules', r'namespaces', r'annotated', r'concepts', r'repo', r'theme' + ): + if nav not in seen_keywords: + seen_keywords.add(nav) + new_navbar.append(nav) + else: + new_navbar.append(nav) + self.navbar = tuple(new_navbar) + self.verbose_value(r'Context.navbar', self.navbar) # tags self.meta_tags = {} @@ -1931,15 +1958,13 @@ def add_internal_asset(p): self.verbose_value(r'Context.html_header', self.html_header) # init emoji db - self.emoji = None - if not self.dry_run: - self.emoji = emoji.Database() + self.emoji = emoji.Database() def __enter__(self): return self def __exit__(self, type, value, traceback): - if not self.dry_run and self.cleanup: + if self.cleanup: delete_directory(self.temp_dir, logger=self.verbose_logger) if not self.xml_only: delete_directory(self.xml_dir, logger=self.verbose_logger) diff --git a/poxy/run.py b/poxy/run.py index cf9249a..b55a1ce 100644 --- a/poxy/run.py +++ b/poxy/run.py @@ -11,17 +11,27 @@ import subprocess import concurrent.futures as futures import tempfile -import requests from lxml import etree from io import BytesIO, StringIO from .utils import * -from . import project +from .project import Context from . import doxygen from . import soup from . import fixers from .svg import SVG from distutils.dir_util import copy_tree +#======================================================================================================================= +# HELPERS +#======================================================================================================================= + + + +def make_temp_file(): + return tempfile.SpooledTemporaryFile(mode='w+', newline='\n', encoding='utf-8') + + + #======================================================================================================================= # PRE/POST PROCESSORS #======================================================================================================================= @@ -141,850 +151,920 @@ -def preprocess_doxyfile(context): +def preprocess_doxyfile(context: Context): assert context is not None - assert isinstance(context, project.Context) + assert isinstance(context, Context) with doxygen.Doxyfile( input_path=None, - output_path=context.doxyfile_path if not context.dry_run else None, + output_path=context.doxyfile_path, cwd=context.input_dir, logger=context.verbose_logger, doxygen_path=context.doxygen_path - ) as df, StringIO(newline='\n') as conf_py: + ) as df: df.append() df.append(r'#---------------------------------------------------------------------------') df.append(r'# marzer/poxy') df.append(r'#---------------------------------------------------------------------------', end='\n\n') - # apply regular doxygen settings - if 1: + df.append(r'# doxygen default overrides', end='\n\n') # ---------------------------------------- - df.append(r'# doxygen default overrides', end='\n\n') # ---------------------------------------- + global _doxygen_overrides + for k, v in _doxygen_overrides: + df.set_value(k, v) - global _doxygen_overrides - for k, v in _doxygen_overrides: - df.set_value(k, v) + df.append() + df.append(r'# general config', end='\n\n') # --------------------------------------------------- + + df.set_value(r'OUTPUT_DIRECTORY', context.output_dir) + df.set_value(r'XML_OUTPUT', context.xml_dir) + df.set_value(r'PROJECT_NAME', context.name) + df.set_value(r'PROJECT_BRIEF', context.description) + df.set_value(r'PROJECT_LOGO', context.logo) + df.set_value(r'SHOW_INCLUDE_FILES', context.show_includes) + df.set_value(r'INTERNAL_DOCS', context.internal_docs) + df.add_value( + r'ENABLED_SECTIONS', (r'private', r'internal') if context.internal_docs else (r'public', r'external') + ) + df.add_value(r'ENABLED_SECTIONS', r'poxy_supports_concepts') - df.append() - df.append(r'# general config', end='\n\n') # --------------------------------------------------- - - df.set_value(r'OUTPUT_DIRECTORY', context.output_dir) - df.set_value(r'XML_OUTPUT', context.xml_dir) - df.set_value(r'PROJECT_NAME', context.name) - df.set_value(r'PROJECT_BRIEF', context.description) - df.set_value(r'PROJECT_LOGO', context.logo) - df.set_value(r'SHOW_INCLUDE_FILES', context.show_includes) - df.set_value(r'INTERNAL_DOCS', context.internal_docs) - df.add_value( - r'ENABLED_SECTIONS', (r'private', r'internal') if context.internal_docs else (r'public', r'external') + if context.generate_tagfile: + context.tagfile_path = Path( + context.output_dir, rf'{context.name.replace(" ","_")}.tagfile.xml' if context.name else r'tagfile.xml' ) - df.add_value(r'ENABLED_SECTIONS', r'poxy_supports_concepts') - - if context.generate_tagfile: - context.tagfile_path = Path( - context.output_dir, - rf'{context.name.replace(" ","_")}.tagfile.xml' if context.name else r'tagfile.xml' - ) - df.set_value(r'GENERATE_TAGFILE', context.tagfile_path.name) - else: - df.set_value(r'GENERATE_TAGFILE', None) + df.set_value(r'GENERATE_TAGFILE', context.tagfile_path.name) + else: + df.set_value(r'GENERATE_TAGFILE', None) + + df.set_value(r'NUM_PROC_THREADS', min(context.threads, 32)) + df.add_value(r'CLANG_OPTIONS', rf'-std=c++{context.cpp%100}') + df.add_value(r'CLANG_OPTIONS', r'-Wno-everything') + + home_md_path = None + for home_md in (r'HOME.md', r'home.md', r'INDEX.md', r'index.md', r'README.md', r'readme.md'): + p = Path(context.input_dir, home_md) + if p.exists() and p.is_file(): + home_md_path = p + break + if home_md_path is not None: + home_md_temp_path = Path(context.temp_pages_dir, r'home.md') + copy_file(home_md_path, home_md_temp_path, logger=context.verbose_logger) + df.set_value(r'USE_MDFILE_AS_MAINPAGE', home_md_temp_path) - df.set_value(r'NUM_PROC_THREADS', min(context.threads, 32)) - df.add_value(r'CLANG_OPTIONS', rf'-std=c++{context.cpp%100}') - df.add_value(r'CLANG_OPTIONS', r'-Wno-everything') + df.append() + df.append(r'# context.warnings', end='\n\n') # --------------------------------------------------- - home_md_path = None - for home_md in (r'HOME.md', r'home.md', r'INDEX.md', r'index.md', r'README.md', r'readme.md'): - p = Path(context.input_dir, home_md) - if p.exists() and p.is_file(): - home_md_path = p - break - if home_md_path is not None: - home_md_temp_path = Path(context.temp_pages_dir, r'home.md') - if not context.dry_run: - copy_file(home_md_path, home_md_temp_path, logger=context.verbose_logger) - df.set_value(r'USE_MDFILE_AS_MAINPAGE', home_md_temp_path) + df.set_value(r'WARNINGS', context.warnings.enabled) + df.set_value(r'WARN_AS_ERROR', False) # we do this ourself + df.set_value(r'WARN_IF_UNDOCUMENTED', context.warnings.undocumented) - df.append() - df.append(r'# context.warnings', end='\n\n') # --------------------------------------------------- + df.append() + df.append(r'# context.sources', end='\n\n') # ---------------------------------------------------- - df.set_value(r'WARNINGS', context.warnings.enabled) - df.set_value(r'WARN_AS_ERROR', False) # we do this ourself - df.set_value(r'WARN_IF_UNDOCUMENTED', context.warnings.undocumented) + df.add_value(r'INPUT', context.sources.paths) + df.set_value(r'FILE_PATTERNS', context.sources.patterns) + df.add_value(r'EXCLUDE', context.html_dir) + df.add_value(r'STRIP_FROM_PATH', context.sources.strip_paths) + df.set_value(r'EXTRACT_ALL', context.sources.extract_all) - df.append() - df.append(r'# context.sources', end='\n\n') # ---------------------------------------------------- + df.append() + df.append(r'# context.examples', end='\n\n') # ---------------------------------------------------- - df.add_value(r'INPUT', context.sources.paths) - df.set_value(r'FILE_PATTERNS', context.sources.patterns) - df.add_value(r'EXCLUDE', context.html_dir) - df.add_value(r'STRIP_FROM_PATH', context.sources.strip_paths) - df.set_value(r'EXTRACT_ALL', context.sources.extract_all) + df.add_value(r'EXAMPLE_PATH', context.examples.paths) + df.set_value(r'EXAMPLE_PATTERNS', context.examples.patterns) + if context.images.paths: # ---------------------------------------------------- df.append() - df.append(r'# context.examples', end='\n\n') # ---------------------------------------------------- - - df.add_value(r'EXAMPLE_PATH', context.examples.paths) - df.set_value(r'EXAMPLE_PATTERNS', context.examples.patterns) + df.append(r'# context.images', end='\n\n') + df.add_value(r'IMAGE_PATH', context.images.paths) - if context.images.paths: # ---------------------------------------------------- - df.append() - df.append(r'# context.images', end='\n\n') - df.add_value(r'IMAGE_PATH', context.images.paths) + if context.tagfiles: # ---------------------------------------------------- + df.append() + df.append(r'# context.tagfiles', end='\n\n') + df.add_value(r'TAGFILES', [rf'{file}={dest}' for _, (file, dest) in context.tagfiles.items()]) - if context.tagfiles: # ---------------------------------------------------- - df.append() - df.append(r'# context.tagfiles', end='\n\n') - df.add_value(r'TAGFILES', [rf'{file}={dest}' for _, (file, dest) in context.tagfiles.items()]) + if context.aliases: # ---------------------------------------------------- + df.append() + df.append(r'# context.aliases', end='\n\n') + df.add_value(r'ALIASES', [rf'{k}={v}' for k, v in context.aliases.items()]) - if context.aliases: # ---------------------------------------------------- - df.append() - df.append(r'# context.aliases', end='\n\n') - df.add_value(r'ALIASES', [rf'{k}={v}' for k, v in context.aliases.items()]) + if context.macros: # ---------------------------------------------------- + df.append() + df.append(r'# context.macros', end='\n\n') + df.add_value(r'PREDEFINED', [rf'{k}={v}' for k, v in context.macros.items()]) - if context.macros: # ---------------------------------------------------- - df.append() - df.append(r'# context.macros', end='\n\n') - df.add_value(r'PREDEFINED', [rf'{k}={v}' for k, v in context.macros.items()]) + df.cleanup() + context.verbose(r'Doxyfile:') + context.verbose(df.get_text(), indent=r' ') - # build HTML_HEADER - html_header = '' - if 1: - # stylesheets - for stylesheet in context.stylesheets: - html_header += f'\n' - # scripts - for script in context.scripts: - html_header += f'\n' - if context.theme != r'custom': - html_header += f'\n' - # metadata - def add_meta_kvp(key_name, key, content): - nonlocal html_header - html_header += f'\n' - - add_meta = lambda key, content: add_meta_kvp(r'name', key, content) - add_property = lambda key, content: add_meta_kvp(r'property', key, content) - add_itemprop = lambda key, content: add_meta_kvp(r'itemprop', key, content) - # metadata - project name - if context.name: - if r'twitter:title' not in context.meta_tags: - add_meta(r'twitter:title', context.name) - add_property(r'og:title', context.name) - add_itemprop(r'name', context.name) - # metadata - project author - if context.author: - if r'author' not in context.meta_tags: - add_meta(r'author', context.author) - add_property(r'article:author', context.author) - # metadata - project description - if context.description: - if r'description' not in context.meta_tags: - add_meta(r'description', context.description) - if r'twitter:description' not in context.meta_tags: - add_meta(r'twitter:description', context.description) - add_property(r'og:description', context.description) - add_itemprop(r'description', context.description) - # metadata - robots - if not context.robots: - if r'robots' not in context.meta_tags: - add_meta(r'robots', r'noindex, nofollow') - if r'googlebot' not in context.meta_tags: - add_meta(r'googlebot', r'noindex, nofollow') - # metadata - misc - if r'format-detection' not in context.meta_tags: - add_meta(r'format-detection', r'telephone=no') - if r'generator' not in context.meta_tags: - add_meta(r'generator', rf'Poxy v{context.version_string}') - if r'referrer' not in context.meta_tags: - add_meta(r'referrer', r'strict-origin-when-cross-origin') - # metadata - additional user-specified tags - for name, content in context.meta_tags.items(): - add_meta(name, content) - # html_header - if context.html_header: - html_header += f'{context.html_header}\n' - html_header = html_header.rstrip() - - # build m.css conf.py - if 1: - conf = lambda s='', end='\n': print(reindent(s, indent=''), file=conf_py, end=end) - conf(rf"DOXYFILE = r'{context.doxyfile_path}'") - conf(r"STYLESHEETS = []") # suppress the default behaviour - conf(rf'HTML_HEADER = """{html_header}"""') - if context.theme == r'dark': - conf(r"THEME_COLOR = '#22272e'") - elif context.theme == r'light': - conf(r"THEME_COLOR = '#cb4b16'") - if not df.contains(r'M_FAVICON'): - if context.favicon: - conf(rf"FAVICON = r'{context.favicon}'") - elif context.theme == r'dark': - conf(rf"FAVICON = 'favicon-dark.png'") - elif context.theme == r'light': - conf(rf"FAVICON = 'favicon-light.png'") - if not df.contains(r'M_SHOW_UNDOCUMENTED'): - conf(rf'SHOW_UNDOCUMENTED = {context.sources.extract_all}') - if not df.contains(r'M_CLASS_TREE_EXPAND_LEVELS'): - conf(r'CLASS_INDEX_EXPAND_LEVELS = 3') - if not df.contains(r'M_FILE_TREE_EXPAND_LEVELS'): - conf(r'FILE_INDEX_EXPAND_LEVELS = 3') - if not df.contains(r'M_EXPAND_INNER_TYPES'): - conf(r'CLASS_INDEX_EXPAND_INNER = True') - if not df.contains(r'M_SEARCH_DOWNLOAD_BINARY'): - conf(r'SEARCH_DOWNLOAD_BINARY = False') - if not df.contains(r'M_SEARCH_DISABLED'): - conf(r'SEARCH_DISABLED = False') - if not df.contains(r'M_LINKS_NAVBAR1') and not df.contains(r'M_LINKS_NAVBAR2'): - navbars = ([], []) - if context.navbar: - bar = [v for v in context.navbar] - for i in range(len(bar)): - if bar[i] == r'repo' and context.repo: - icon_path = Path(dirs.DATA, context.repo.icon_filename) - if icon_path.exists(): - svg = SVG(icon_path, logger=context.verbose_logger, root_id=r'poxy-repo-icon') - bar[i] = ( - rf'{svg}', [] - ) - else: - bar[i] = None - elif bar[i] == r'theme': - svg = SVG( - Path(dirs.DATA, r'poxy-icon-theme.svg'), - logger=context.verbose_logger, - root_id=r'poxy-theme-switch-img' - ) - bar[i] = ( - r'{svg}', [] - ) - bar = [b for b in bar if b is not None] - split = min(max(int(len(bar) / 2) + len(bar) % 2, 2), len(bar)) - for b, i in ((bar[:split], 0), (bar[split:], 1)): - for j in range(len(b)): - if isinstance(b[j], tuple): - navbars[i].append(b[j]) - else: - navbars[i].append((None, b[j], [])) - for i in (0, 1): - if navbars[i]: - conf(f'LINKS_NAVBAR{i+1} = [\n\t', end='') - conf(',\n\t'.join([rf'{b}' for b in navbars[i]])) - conf(r']') - else: - conf(rf'LINKS_NAVBAR{i+1} = []') - if not df.contains(r'M_PAGE_FINE_PRINT'): - conf(r"FINE_PRINT = r'''") - footer = [] - if context.repo: - footer.append(rf'{type(context.repo).__name__}') - footer.append(rf'Report an issue') - if context.changelog: - footer.append(rf'Changelog') - if context.license and context.license[r'uri']: - footer.append(rf'License') - if context.generate_tagfile: - footer.append( - rf'Doxygen tagfile' - ) - if footer: - for i in range(1, len(footer)): - footer[i] = r' • ' + footer[i] - footer.append(r'

') - footer.append(r'Site generated using Poxy') - for i in range(len(footer)): - conf(rf" {footer[i]}") - conf(r"'''") - conf_py_text = conf_py.getvalue() - # write conf.py - if not context.dry_run: - context.verbose(rf'Writing {context.mcss_conf_path}') - with open(context.mcss_conf_path, r'w', encoding=r'utf-8', newline='\n') as f: - f.write(conf_py_text) +def preprocess_changelog(context: Context): + assert context is not None + assert isinstance(context, Context) + if not context.changelog: + return - # clean and debug dump final doxyfile - df.cleanup() - if context.dry_run: - context.info(r'#====================================================================================') - context.info(rf'# generated by Poxy v{context.version_string}') - context.info(r'#====================================================================================') - context.info(df.get_text()) - context.info(r'## ---------------------------------------------------------------------------------') - context.info(r'## m.css conf.py:') - context.info(r'## ---------------------------------------------------------------------------------') - context.info(conf_py_text, indent='## ') - context.info(r'#====================================================================================') - else: - context.verbose(r'Effective Doxyfile:') - context.verbose(df.get_text(), indent=r' ') - context.verbose(r' ## --------------------------------------------------------------------------') - context.verbose(r' ## m.css conf.py:') - context.verbose(r' ## --------------------------------------------------------------------------') - context.verbose(conf_py_text, indent='## ') + # make sure we're working with a temp copy, not the user's actual changelog + # (the actual copying should already be done in the context's initialization) + assert context.changelog.parent == context.temp_pages_dir + assert_existing_file(context.changelog) + + text = read_all_text_from_file(context.changelog, logger=context.verbose_logger).strip() + text = text.replace('\r\n', '\n') + text = re.sub(r'\n\n', r'', text) + if context.repo: + text = re.sub(r'#([0-9]+)', lambda m: rf'[#{m[1]}]({context.repo.make_issue_uri(m[1])})', text) + text = re.sub(r'!([0-9]+)', lambda m: rf'[!{m[1]}]({context.repo.make_pull_request_uri(m[1])})', text) + text = re.sub(r'@([a-zA-Z0-9_-]+)', lambda m: rf'[@{m[1]}]({context.repo.make_user_uri(m[1])})', text) + text = text.replace(r'&', r'__poxy_thiswasan_amp') + text = text.replace(r'️', r'__poxy_thiswasan_fe0f') + text = text.replace(r'@', r'__poxy_thiswasan_at') + if text.find(r'@tableofcontents') == -1 and text.find('\\tableofcontents') == -1 and text.find(r'[TOC]') == -1: + #text = f'[TOC]\n\n{text}' + nlnl = text.find(r'\n\n') + if nlnl != -1: + text = f'{text[:nlnl]}\n\n\\tableofcontents\n\n{text[nlnl:]}' + pass + text += '\n\n' + context.verbose(rf'Writing {context.changelog}') + with open(context.changelog, r'w', encoding=r'utf-8', newline='\n') as f: + f.write(text) + + + +def preprocess_tagfiles(context: Context): + assert context is not None + assert isinstance(context, Context) + if not context.unresolved_tagfiles: + return + with ScopeTimer(r'Resolving remote tagfiles', print_start=True, print_end=context.verbose_logger) as t: + for source, (file, _) in context.tagfiles.items(): + if file.exists() or not is_uri(source): + continue + context.verbose(rf'Downloading {source}') + text = download_text(source, timeout=30) + context.verbose(rf'Writing {file}') + with open(file, 'w', encoding='utf-8', newline='\n') as f: + f.write(text) -def postprocess_xml(context): +def postprocess_xml(context: Context): assert context is not None - assert isinstance(context, project.Context) + assert isinstance(context, Context) xml_files = get_all_files(context.xml_dir, any=(r'*.xml')) if not xml_files: return - with ScopeTimer( - rf'Post-processing {len(xml_files) + len(context.tagfiles)} XML files', - print_start=True, - print_end=context.verbose_logger - ): + context.info(rf'Post-processing {len(xml_files) + len(context.tagfiles)} XML files...') - pretty_print_xml = False - xml_parser = etree.XMLParser( - encoding='utf-8', remove_blank_text=pretty_print_xml, recover=True, remove_comments=True, ns_clean=True - ) - write_xml_to_file = lambda xml, f: xml.write( - str(f), encoding='utf-8', xml_declaration=True, pretty_print=pretty_print_xml - ) + pretty_print_xml = False + xml_parser = etree.XMLParser( + encoding='utf-8', remove_blank_text=pretty_print_xml, recover=True, remove_comments=True, ns_clean=True + ) + write_xml_to_file = lambda xml, f: xml.write( + str(f), encoding='utf-8', xml_declaration=True, pretty_print=pretty_print_xml + ) - inline_namespace_ids = None - if context.inline_namespaces: - inline_namespace_ids = [f'namespace{doxygen.mangle_name(ns)}' for ns in context.inline_namespaces] - - implementation_header_data = None - implementation_header_mappings = None - implementation_header_innernamespaces = None - implementation_header_sectiondefs = None - implementation_header_unused_keys = None - implementation_header_unused_values = None - if context.implementation_headers: - implementation_header_data = [( - hp, os.path.basename(hp), doxygen.mangle_name(os.path.basename(hp)), - [(i, os.path.basename(i), doxygen.mangle_name(os.path.basename(i))) for i in impl] - ) for hp, impl in context.implementation_headers] - implementation_header_unused_keys = set() - for hp, impl in context.implementation_headers: - implementation_header_unused_keys.add(hp) - implementation_header_unused_values = dict() - for hdata in implementation_header_data: - for (ip, ifn, iid) in hdata[3]: - implementation_header_unused_values[iid] = (ip, hdata[0]) - implementation_header_mappings = dict() - implementation_header_innernamespaces = dict() - implementation_header_sectiondefs = dict() - for hdata in implementation_header_data: - implementation_header_innernamespaces[hdata[2]] = [] - implementation_header_sectiondefs[hdata[2]] = [] - for (ip, ifn, iid) in hdata[3]: - implementation_header_mappings[iid] = hdata - - # process xml files + inline_namespace_ids = None + if context.inline_namespaces: + inline_namespace_ids = [f'namespace{doxygen.mangle_name(ns)}' for ns in context.inline_namespaces] + + implementation_header_data = None + implementation_header_mappings = None + implementation_header_innernamespaces = None + implementation_header_sectiondefs = None + implementation_header_unused_keys = None + implementation_header_unused_values = None + if context.implementation_headers: + implementation_header_data = [( + hp, os.path.basename(hp), doxygen.mangle_name(os.path.basename(hp)), + [(i, os.path.basename(i), doxygen.mangle_name(os.path.basename(i))) for i in impl] + ) for hp, impl in context.implementation_headers] + implementation_header_unused_keys = set() + for hp, impl in context.implementation_headers: + implementation_header_unused_keys.add(hp) + implementation_header_unused_values = dict() + for hdata in implementation_header_data: + for (ip, ifn, iid) in hdata[3]: + implementation_header_unused_values[iid] = (ip, hdata[0]) + implementation_header_mappings = dict() + implementation_header_innernamespaces = dict() + implementation_header_sectiondefs = dict() + for hdata in implementation_header_data: + implementation_header_innernamespaces[hdata[2]] = [] + implementation_header_sectiondefs[hdata[2]] = [] + for (ip, ifn, iid) in hdata[3]: + implementation_header_mappings[iid] = hdata + + setattr(context, r'compound_pages', dict()) + setattr(context, r'compound_kinds', set()) + + # process xml files + if 1: + + # pre-pass to delete junk files if 1: + # 'file' entries for markdown and dox files + dox_files = [rf'*{doxygen.mangle_name(ext)}.xml' for ext in (r'.dox', r'.md')] + dox_files.append(r'md_home.xml') + for xml_file in get_all_files(context.xml_dir, any=dox_files): + delete_file(xml_file, logger=context.verbose_logger) + + # 'dir' entries for empty directories + deleted = True + while deleted: + deleted = False + for xml_file in get_all_files(context.xml_dir, all=(r'dir*.xml')): + xml = etree.parse(str(xml_file), parser=xml_parser) + compounddef = xml.getroot().find(r'compounddef') + if compounddef is None or compounddef.get(r'kind') != r'dir': + continue + existing_inners = 0 + for subtype in (r'innerfile', r'innerdir'): + for inner in compounddef.findall(subtype): + ref_file = Path(context.xml_dir, rf'{inner.get(r"refid")}.xml') + if ref_file.exists(): + existing_inners = existing_inners + 1 + if not existing_inners: + delete_file(xml_file, logger=context.verbose_logger) + deleted = True + + extracted_implementation = False + tentative_macros = regex_or(context.code_blocks.macros) + macros = set() + cpp_tree = CppTree() + xml_files = get_all_files(context.xml_dir, any=(r'*.xml')) + tagfiles = [f for _, (f, _) in context.tagfiles.items()] + xml_files = xml_files + tagfiles + all_inners_by_type = {r'namespace': set(), r'class': set(), r'concept': set()} + for xml_file in xml_files: + + context.verbose(rf'Pre-processing {xml_file}') + if xml_file.name == r'Doxyfile.xml': + continue + + xml = etree.parse(str(xml_file), parser=xml_parser) + root = xml.getroot() + changed = False + + # the doxygen index + if root.tag == r'doxygenindex': + + # remove entries for files we might have explicitly deleted above + for compound in [ + tag for tag in root.findall(r'compound') if tag.get(r'kind') in (r'file', r'dir', r'concept') + ]: + ref_file = Path(context.xml_dir, rf'{compound.get(r"refid")}.xml') + if not ref_file.exists(): + root.remove(compound) + changed = True + + # extract namespaces, types and enum values for syntax highlighting + scopes = [ + tag for tag in root.findall(r'compound') + if tag.get(r'kind') in (r'namespace', r'class', r'struct', r'union') + ] + for scope in scopes: + scope_name = scope.find(r'name').text + + # skip template members because they'll break the regex matchers + if scope_name.find(r'<') != -1: + continue - # pre-pass to delete junk files - if 1: - # 'file' entries for markdown and dox files - dox_files = [rf'*{doxygen.mangle_name(ext)}.xml' for ext in (r'.dox', r'.md')] - dox_files.append(r'md_home.xml') - for xml_file in get_all_files(context.xml_dir, any=dox_files): - delete_file(xml_file, logger=context.verbose_logger) - - # 'dir' entries for empty directories - deleted = True - while deleted: - deleted = False - for xml_file in get_all_files(context.xml_dir, all=(r'dir*.xml')): - xml = etree.parse(str(xml_file), parser=xml_parser) - compounddef = xml.getroot().find(r'compounddef') - if compounddef is None or compounddef.get(r'kind') != r'dir': - continue - existing_inners = 0 - for subtype in (r'innerfile', r'innerdir'): - for inner in compounddef.findall(subtype): - ref_file = Path(context.xml_dir, rf'{inner.get(r"refid")}.xml') - if ref_file.exists(): - existing_inners = existing_inners + 1 - if not existing_inners: - delete_file(xml_file, logger=context.verbose_logger) - deleted = True - - extracted_implementation = False - tentative_macros = regex_or(context.code_blocks.macros) - macros = set() - cpp_tree = CppTree() - xml_files = get_all_files(context.xml_dir, any=(r'*.xml')) - tagfiles = [f for _, (f, _) in context.tagfiles.items()] - xml_files = xml_files + tagfiles - all_inners_by_type = {r'namespace': set(), r'class': set(), r'concept': set()} - for xml_file in xml_files: - - context.verbose(rf'Pre-processing {xml_file}') - if xml_file.name == r'Doxyfile.xml': - continue - - xml = etree.parse(str(xml_file), parser=xml_parser) - root = xml.getroot() - changed = False - - # the doxygen index - if root.tag == r'doxygenindex': + # regular types and namespaces + if scope.get(r'kind') in (r'class', r'struct', r'union'): + cpp_tree.add_type(scope_name) + elif scope.get(r'kind') == r'namespace': + cpp_tree.add_namespace(scope_name) + + # nested enums + enum_tags = [tag for tag in scope.findall(r'member') if tag.get(r'kind') in (r'enum', r'enumvalue')] + enum_name = '' + for tag in enum_tags: + if tag.get(r'kind') == r'enum': + enum_name = rf'{scope_name}::{tag.find("name").text}' + cpp_tree.add_type(enum_name) + else: + assert enum_name + cpp_tree.add_enum_value(rf'{enum_name}::{tag.find("name").text}') + + # nested typedefs + typedefs = [tag for tag in scope.findall(r'member') if tag.get(r'kind') == r'typedef'] + for typedef in typedefs: + cpp_tree.add_type(rf'{scope_name}::{typedef.find("name").text}') + + # enumerate all compound pages and their types for later (e.g. HTML post-process) + for tag in root.findall(r'compound'): + refid = tag.get(r'refid') + filename = refid + if filename == r'indexpage': + filename = r'index' + filename = filename + r'.html' + context.compound_pages[filename] = { + r'kind': tag.get(r'kind'), + r'name': tag.find(r'name').text, + r'refid': refid + } + context.compound_kinds.add(tag.get(r'kind')) + context.verbose_value(r'Context.compound_pages', context.compound_pages) + context.verbose_value(r'Context.compound_kinds', context.compound_kinds) + + # a tag file + elif root.tag == r'tagfile': + for compound in [ + tag for tag in root.findall(r'compound') + if tag.get(r'kind') in (r'namespace', r'class', r'struct', r'union', r'concept') + ]: + + compound_name = compound.find(r'name').text + if compound_name.find(r'<') != -1: + continue - # remove entries for files we might have explicitly deleted above - for compound in [ - tag for tag in root.findall(r'compound') if tag.get(r'kind') in (r'file', r'dir', r'concept') - ]: - ref_file = Path(context.xml_dir, rf'{compound.get(r"refid")}.xml') - if not ref_file.exists(): - root.remove(compound) - changed = True - - # extract namespaces, types and enum values for syntax highlighting - scopes = [ - tag for tag in root.findall(r'compound') - if tag.get(r'kind') in (r'namespace', r'class', r'struct', r'union') - ] - for scope in scopes: - scope_name = scope.find(r'name').text - - # skip template members because they'll break the regex matchers - if scope_name.find(r'<') != -1: - continue + compound_type = compound.get(r'kind') + if compound_type in (r'class', r'struct', r'union', r'concept'): + cpp_tree.add_type(compound_name) + else: + cpp_tree.add_namespace(compound_name) - # regular types and namespaces - if scope.get(r'kind') in (r'class', r'struct', r'union'): - cpp_tree.add_type(scope_name) - elif scope.get(r'kind') == r'namespace': - cpp_tree.add_namespace(scope_name) - - # nested enums - enum_tags = [ - tag for tag in scope.findall(r'member') if tag.get(r'kind') in (r'enum', r'enumvalue') - ] - enum_name = '' - for tag in enum_tags: - if tag.get(r'kind') == r'enum': - enum_name = rf'{scope_name}::{tag.find("name").text}' - cpp_tree.add_type(enum_name) - else: - assert enum_name - cpp_tree.add_enum_value(rf'{enum_name}::{tag.find("name").text}') - - # nested typedefs - typedefs = [tag for tag in scope.findall(r'member') if tag.get(r'kind') == r'typedef'] - for typedef in typedefs: - cpp_tree.add_type(rf'{scope_name}::{typedef.find("name").text}') - - # enumerate all compound pages and their types for use later in the HTML post-process - pages = {} - for tag in root.findall(r'compound'): - refid = tag.get(r'refid') - filename = refid - if filename == r'indexpage': - filename = r'index' - filename = filename + r'.html' - pages[filename] = {r'kind': tag.get(r'kind'), r'name': tag.find(r'name').text, r'refid': refid} - context.__dict__[r'compound_pages'] = pages - context.verbose_value(r'Context.compound_pages', pages) - - # a tag file - elif root.tag == r'tagfile': - for compound in [ - tag for tag in root.findall(r'compound') + for member in [ + tag for tag in compound.findall(r'member') if tag.get(r'kind') in (r'namespace', r'class', r'struct', r'union', r'concept') ]: - compound_name = compound.find(r'name').text - if compound_name.find(r'<') != -1: + member_name = member.find(r'name').text + if member_name.find(r'<') != -1: continue - compound_type = compound.get(r'kind') - if compound_type in (r'class', r'struct', r'union', r'concept'): + member_type = member.get(r'kind') + if member_type in (r'class', r'struct', r'union', r'concept'): cpp_tree.add_type(compound_name) else: cpp_tree.add_namespace(compound_name) - for member in [ - tag for tag in compound.findall(r'member') - if tag.get(r'kind') in (r'namespace', r'class', r'struct', r'union', r'concept') - ]: - - member_name = member.find(r'name').text - if member_name.find(r'<') != -1: - continue - - member_type = member.get(r'kind') - if member_type in (r'class', r'struct', r'union', r'concept'): - cpp_tree.add_type(compound_name) - else: - cpp_tree.add_namespace(compound_name) - - # some other compound definition - else: - compounddef = root.find(r'compounddef') - if compounddef is None: - context.warning(rf'{xml_file} did not contain a !') - continue + # some other compound definition + else: + compounddef = root.find(r'compounddef') + if compounddef is None: + context.warning(rf'{xml_file} did not contain a !') + continue - compound_id = compounddef.get(r'id') - if compound_id is None or not compound_id: - context.warning(rf'{xml_file} did not have attribute "id"!') - continue + compound_id = compounddef.get(r'id') + if compound_id is None or not compound_id: + context.warning(rf'{xml_file} did not have attribute "id"!') + continue - compound_kind = compounddef.get(r'kind') - if compound_kind is None or not compound_kind: - context.warning(rf'{xml_file} did not have attribute "kind"!') - continue + compound_kind = compounddef.get(r'kind') + if compound_kind is None or not compound_kind: + context.warning(rf'{xml_file} did not have attribute "kind"!') + continue - compound_name = compounddef.find(r'compoundname') - if compound_name is None or not compound_name.text: - context.warning(rf'{xml_file} did not contain a valid !') - continue - compound_name = str(compound_name.text).strip() - - if compound_kind in ( - r'namespace', r'class', r'struct', r'union', r'enum', r'file', r'group', r'concept' - ): - - # merge user-defined sections with the same name - sectiondefs = [ - s for s in compounddef.findall(r'sectiondef') if s.get(r'kind') == r'user-defined' - ] - sections = dict() - for section in sectiondefs: - header = section.find(r'header') - if header is not None and header.text: - if header.text not in sections: - sections[header.text] = [] - sections[header.text].append(section) - for key, vals in sections.items(): - if len(vals) > 1: - first_section = vals.pop(0) - for section in vals: - for member in section.findall(r'memberdef'): - section.remove(member) - first_section.append(member) - compounddef.remove(section) + compound_name = compounddef.find(r'compoundname') + if compound_name is None or not compound_name.text: + context.warning(rf'{xml_file} did not contain a valid !') + continue + compound_name = str(compound_name.text).strip() + + if compound_kind != r'page': + + # merge user-defined sections with the same name + sectiondefs = [s for s in compounddef.findall(r'sectiondef') if s.get(r'kind') == r'user-defined'] + sections = dict() + for section in sectiondefs: + header = section.find(r'header') + if header is not None and header.text: + if header.text not in sections: + sections[header.text] = [] + sections[header.text].append(section) + for key, vals in sections.items(): + if len(vals) > 1: + first_section = vals.pop(0) + for section in vals: + for member in section.findall(r'memberdef'): + section.remove(member) + first_section.append(member) + compounddef.remove(section) + changed = True + + # sort user-defined sections based on their name + sectiondefs = [s for s in compounddef.findall(r'sectiondef') if s.get(r'kind') == r'user-defined'] + sectiondefs = [s for s in sectiondefs if s.find(r'header') is not None] + for section in sectiondefs: + compounddef.remove(section) + sectiondefs.sort(key=lambda s: s.find(r'header').text) + for section in sectiondefs: + compounddef.append(section) + changed = True + + # per-section stuff + for section in compounddef.findall(r'sectiondef'): + + # remove members which are listed multiple times because doxygen is idiotic: + members = [tag for tag in section.findall(r'memberdef')] + for i in range(len(members) - 1, 0, -1): + for j in range(i): + if members[i].get(r'id') == members[j].get(r'id'): + section.remove(members[i]) changed = True + break - # sort user-defined sections based on their name - sectiondefs = [ - s for s in compounddef.findall(r'sectiondef') if s.get(r'kind') == r'user-defined' - ] - sectiondefs = [s for s in sectiondefs if s.find(r'header') is not None] - for section in sectiondefs: - compounddef.remove(section) - sectiondefs.sort(key=lambda s: s.find(r'header').text) - for section in sectiondefs: - compounddef.append(section) - changed = True - - # per-section stuff - for section in compounddef.findall(r'sectiondef'): - - # remove members which are listed multiple times because doxygen is idiotic: - members = [tag for tag in section.findall(r'memberdef')] - for i in range(len(members) - 1, 0, -1): - for j in range(i): - if members[i].get(r'id') == members[j].get(r'id'): - section.remove(members[i]) - changed = True - break - - # fix functions where keywords like 'friend' have been erroneously included in the return type - if 1: - members = [ - m for m in section.findall(r'memberdef') - if m.get(r'kind') in (r'friend', r'function') - ] - attribute_keywords = ((r'constexpr', r'constexpr', - r'yes'), (r'consteval', r'consteval', r'yes'), (r'explicit', r'explicit', - r'yes'), (r'static', r'static', r'yes'), (r'friend', None, None), - (r'inline', r'inline', r'yes'), (r'virtual', r'virt', r'virtual')) - for member in members: - type = member.find(r'type') - if type is None or type.text is None: - continue - matched_bad_keyword = True - while matched_bad_keyword: - matched_bad_keyword = False - for kw, attr, attr_value in attribute_keywords: - if type.text == kw: # constructors - type.text = '' - elif type.text.startswith(kw + ' '): - type.text = type.text[len(kw):].strip() - elif type.text.endswith(' ' + kw): - type.text = type.text[:len(kw)].strip() - else: - continue - matched_bad_keyword = True - changed = True - if attr is not None: - member.set(attr, attr_value) - elif kw == r'friend': - member.set(r'kind', r'friend') - - # re-sort members to override Doxygen's weird and stupid sorting 'rules' - if 1: - sort_members_by_name = lambda tag: tag.find(r'name').text - members = [tag for tag in section.findall(r'memberdef')] - for tag in members: - section.remove(tag) - # fmt: off - # yapf: disable - groups = [ - ([tag for tag in members if tag.get(r'kind') == r'define'], True), # - ([tag for tag in members if tag.get(r'kind') == r'typedef'], True), - ([tag for tag in members if tag.get(r'kind') == r'concept'], True), - ([tag for tag in members if tag.get(r'kind') == r'enum'], True), - ([tag for tag in members if tag.get(r'kind') == r'variable' and tag.get(r'static') == r'yes'], True), - ([tag for tag in members if tag.get(r'kind') == r'variable' and tag.get(r'static') == r'no'], compound_kind not in (r'class', r'struct', r'union')), - ([tag for tag in members if tag.get(r'kind') == r'function' and tag.get(r'static') == r'yes'], True), - ([tag for tag in members if tag.get(r'kind') == r'function' and tag.get(r'static') == r'no'], True), - ([tag for tag in members if tag.get(r'kind') == r'friend'], True) - ] - # yapf: enable - # fmt: on - for group, sort in groups: - if sort: - group.sort(key=sort_members_by_name) - for tag in group: - members.remove(tag) - section.append(tag) + # fix functions where keywords like 'friend' have been erroneously included in the return type + if 1: + members = [ + m for m in section.findall(r'memberdef') if m.get(r'kind') in (r'friend', r'function') + ] + attribute_keywords = ((r'constexpr', r'constexpr', + r'yes'), (r'consteval', r'consteval', r'yes'), (r'explicit', r'explicit', r'yes'), + (r'static', r'static', r'yes'), (r'friend', None, None), (r'inline', r'inline', + r'yes'), (r'virtual', r'virt', r'virtual')) + for member in members: + type = member.find(r'type') + if type is None or type.text is None: + continue + matched_bad_keyword = True + while matched_bad_keyword: + matched_bad_keyword = False + for kw, attr, attr_value in attribute_keywords: + if type.text == kw: # constructors + type.text = '' + elif type.text.startswith(kw + ' '): + type.text = type.text[len(kw):].strip() + elif type.text.endswith(' ' + kw): + type.text = type.text[:len(kw)].strip() + else: + continue + matched_bad_keyword = True changed = True - # if we've missed any groups just glob them on the end - if members: - members.sort(key=sort_members_by_name) + if attr is not None: + member.set(attr, attr_value) + elif kw == r'friend': + member.set(r'kind', r'friend') + + # re-sort members to override Doxygen's weird and stupid sorting 'rules' + if 1: + sort_members_by_name = lambda tag: tag.find(r'name').text + members = [tag for tag in section.findall(r'memberdef')] + for tag in members: + section.remove(tag) + # fmt: off + # yapf: disable + groups = [ + ([tag for tag in members if tag.get(r'kind') == r'define'], True), # + ([tag for tag in members if tag.get(r'kind') == r'typedef'], True), + ([tag for tag in members if tag.get(r'kind') == r'concept'], True), + ([tag for tag in members if tag.get(r'kind') == r'enum'], True), + ([tag for tag in members if tag.get(r'kind') == r'variable' and tag.get(r'static') == r'yes'], True), + ([tag for tag in members if tag.get(r'kind') == r'variable' and tag.get(r'static') == r'no'], compound_kind not in (r'class', r'struct', r'union')), + ([tag for tag in members if tag.get(r'kind') == r'function' and tag.get(r'static') == r'yes'], True), + ([tag for tag in members if tag.get(r'kind') == r'function' and tag.get(r'static') == r'no'], True), + ([tag for tag in members if tag.get(r'kind') == r'friend'], True) + ] + # yapf: enable + # fmt: on + for group, sort in groups: + if sort: + group.sort(key=sort_members_by_name) + for tag in group: + members.remove(tag) + section.append(tag) changed = True - for tag in members: - section.append(tag) - - # namespaces - if compound_kind == r'namespace': + # if we've missed any groups just glob them on the end + if members: + members.sort(key=sort_members_by_name) + changed = True + for tag in members: + section.append(tag) - # set inline namespaces - if context.inline_namespaces: - for nsid in inline_namespace_ids: - if compound_id == nsid: - compounddef.set(r'inline', r'yes') - changed = True - break + # namespaces + if compound_kind == r'namespace': - # dirs - if compound_kind == r'dir': + # set inline namespaces + if context.inline_namespaces: + for nsid in inline_namespace_ids: + if compound_id == nsid: + compounddef.set(r'inline', r'yes') + changed = True + break - # remove implementation headers - if context.implementation_headers: - for innerfile in compounddef.findall(r'innerfile'): - if innerfile.get(r'refid') in implementation_header_mappings: - compounddef.remove(innerfile) + # dirs + if compound_kind == r'dir': + + # remove implementation headers + if context.implementation_headers: + for innerfile in compounddef.findall(r'innerfile'): + if innerfile.get(r'refid') in implementation_header_mappings: + compounddef.remove(innerfile) + changed = True + + # files + if compound_kind == r'file': + + # simplify the XML by removing junk not used by mcss + if not context.xml_only: + for tag in (r'includes', r'includedby', r'incdepgraph', r'invincdepgraph'): + for t in compounddef.findall(tag): + compounddef.remove(t) + changed = True + + # get any macros for the syntax highlighter + for sectiondef in [ + tag for tag in compounddef.findall(r'sectiondef') if tag.get(r'kind') == r'define' + ]: + for memberdef in [ + tag for tag in sectiondef.findall(r'memberdef') if tag.get(r'kind') == r'define' + ]: + macro = memberdef.find(r'name').text + if not tentative_macros.fullmatch(macro): + macros.add(macro) + + # rip the good bits out of implementation headers + if context.implementation_headers: + iid = compound_id + if iid in implementation_header_mappings: + hid = implementation_header_mappings[iid][2] + innernamespaces = compounddef.findall(r'innernamespace') + if innernamespaces: + implementation_header_innernamespaces[ + hid] = implementation_header_innernamespaces[hid] + innernamespaces + extracted_implementation = True + if iid in implementation_header_unused_values: + del implementation_header_unused_values[iid] + for tag in innernamespaces: + compounddef.remove(tag) changed = True - - # files - if compound_kind == r'file': - - # simplify the XML by removing junk not used by mcss - if not context.xml_only: - for tag in (r'includes', r'includedby', r'incdepgraph', r'invincdepgraph'): - for t in compounddef.findall(tag): - compounddef.remove(t) + sectiondefs = compounddef.findall(r'sectiondef') + if sectiondefs: + implementation_header_sectiondefs[ + hid] = implementation_header_sectiondefs[hid] + sectiondefs + extracted_implementation = True + if iid in implementation_header_unused_values: + del implementation_header_unused_values[iid] + for tag in sectiondefs: + compounddef.remove(tag) changed = True - # get any macros for the syntax highlighter - for sectiondef in [ - tag for tag in compounddef.findall(r'sectiondef') if tag.get(r'kind') == r'define' - ]: - for memberdef in [ - tag for tag in sectiondef.findall(r'memberdef') if tag.get(r'kind') == r'define' - ]: - macro = memberdef.find(r'name').text - if not tentative_macros.fullmatch(macro): - macros.add(macro) - - # rip the good bits out of implementation headers - if context.implementation_headers: - iid = compound_id - if iid in implementation_header_mappings: - hid = implementation_header_mappings[iid][2] - innernamespaces = compounddef.findall(r'innernamespace') - if innernamespaces: - implementation_header_innernamespaces[ - hid] = implementation_header_innernamespaces[hid] + innernamespaces - extracted_implementation = True - if iid in implementation_header_unused_values: - del implementation_header_unused_values[iid] - for tag in innernamespaces: - compounddef.remove(tag) - changed = True - sectiondefs = compounddef.findall(r'sectiondef') - if sectiondefs: - implementation_header_sectiondefs[ - hid] = implementation_header_sectiondefs[hid] + sectiondefs - extracted_implementation = True - if iid in implementation_header_unused_values: - del implementation_header_unused_values[iid] - for tag in sectiondefs: - compounddef.remove(tag) - changed = True + # groups and namespaces + if compound_kind in (r'group', r'namespace'): + + # fix inner(class|namespace|group|concept) sorting + inners = [tag for tag in compounddef.iterchildren() if tag.tag.startswith(r'inner')] + if inners: + changed = True + for tag in inners: + compounddef.remove(tag) + inners.sort(key=lambda tag: tag.text) + for tag in inners: + compounddef.append(tag) + + # all namespace 'innerXXXXXX' + if compound_kind in (r'namespace', r'struct', r'class', r'union', r'concept'): + if compound_name.rfind(r'::') != -1: + all_inners_by_type[r'class' if compound_kind in (r'struct', r'union') else compound_kind].add( + (compound_id, compound_name) + ) + + if changed and xml_file not in tagfiles: # tagfiles are read-only - ensure we don't modify them + write_xml_to_file(xml, xml_file) - # groups and namespaces - if compound_kind in (r'group', r'namespace'): - - # fix inner(class|namespace|group|concept) sorting - inners = [tag for tag in compounddef.iterchildren() if tag.tag.startswith(r'inner')] - if inners: - changed = True - for tag in inners: - compounddef.remove(tag) - inners.sort(key=lambda tag: tag.text) - for tag in inners: - compounddef.append(tag) - - # all namespace 'innerXXXXXX' - if compound_kind in (r'namespace', r'struct', r'class', r'union', r'concept'): - if compound_name.rfind(r'::') != -1: - all_inners_by_type[r'class' if compound_kind in (r'struct', - r'union') else compound_kind].add((compound_id, compound_name)) + # add to syntax highlighter + context.code_blocks.namespaces.add(cpp_tree.matcher(CppTree.NAMESPACES)) + context.code_blocks.types.add(cpp_tree.matcher(CppTree.TYPES)) + context.code_blocks.enums.add(cpp_tree.matcher(CppTree.ENUM_VALUES)) + for macro in macros: + context.code_blocks.macros.add(macro) + context.verbose_object(r'Context.code_blocks', context.code_blocks) + # fix up namespaces/classes that are missing nodes + if 1: + outer_namespaces = dict() + for inner_type, ids_and_names in all_inners_by_type.items(): + for id, name in ids_and_names: + ns = name[:name.rfind(r'::')] + assert ns + if ns not in outer_namespaces: + outer_namespaces[ns] = [] + outer_namespaces[ns].append((inner_type, id, name)) + for ns, vals in outer_namespaces.items(): + xml_file = None + for outer_type in (r'namespace', r'struct', r'class', r'union'): + f = Path(context.xml_dir, rf'{outer_type}{doxygen.mangle_name(ns)}.xml') + if f.exists(): + xml_file = f + break + if not xml_file: + continue + xml = etree.parse(str(xml_file), parser=xml_parser) + compounddef = xml.getroot().find(r'compounddef') + if compounddef is None: + continue + changed = False + existing_inner_ids = set() + for inner_type in (r'class', r'namespace', r'concept'): + for elem in compounddef.findall(rf'inner{inner_type}'): + id = elem.get(r'refid') + if id: + existing_inner_ids.add(str(id)) + for (inner_type, id, name) in vals: + if id not in existing_inner_ids: + elem = etree.SubElement(compounddef, rf'inner{inner_type}') + elem.text = name + elem.set(r'refid', id) + elem.set(r'prot', r'public') # todo: this isn't necessarily correct + existing_inner_ids.add(id) + changed = True if changed: write_xml_to_file(xml, xml_file) - # add to syntax highlighter - context.code_blocks.namespaces.add(cpp_tree.matcher(CppTree.NAMESPACES)) - context.code_blocks.types.add(cpp_tree.matcher(CppTree.TYPES)) - context.code_blocks.enums.add(cpp_tree.matcher(CppTree.ENUM_VALUES)) - for macro in macros: - context.code_blocks.macros.add(macro) - - # fix up namespaces/classes that are missing nodes - if 1: - outer_namespaces = dict() - for inner_type, ids_and_names in all_inners_by_type.items(): - for id, name in ids_and_names: - ns = name[:name.rfind(r'::')] - assert ns - if ns not in outer_namespaces: - outer_namespaces[ns] = [] - outer_namespaces[ns].append((inner_type, id, name)) - for ns, vals in outer_namespaces.items(): - xml_file = None - for outer_type in (r'namespace', r'struct', r'class', r'union'): - f = Path(context.xml_dir, rf'{outer_type}{doxygen.mangle_name(ns)}.xml') - if f.exists(): - xml_file = f + # merge extracted implementations + if extracted_implementation: + for (hp, hfn, hid, impl) in implementation_header_data: + xml_file = Path(context.xml_dir, rf'{hid}.xml') + context.verbose(rf'Merging implementation nodes into {xml_file}') + xml = etree.parse(str(xml_file), parser=xml_parser) + compounddef = xml.getroot().find(r'compounddef') + changed = False + + innernamespaces = compounddef.findall(r'innernamespace') + for new_tag in implementation_header_innernamespaces[hid]: + matched = False + for existing_tag in innernamespaces: + if existing_tag.get(r'refid') == new_tag.get(r'refid'): + matched = True break - if not xml_file: - continue - xml = etree.parse(str(xml_file), parser=xml_parser) - compounddef = xml.getroot().find(r'compounddef') - if compounddef is None: - continue - changed = False - existing_inner_ids = set() - for inner_type in (r'class', r'namespace', r'concept'): - for elem in compounddef.findall(rf'inner{inner_type}'): - id = elem.get(r'refid') - if id: - existing_inner_ids.add(str(id)) - for (inner_type, id, name) in vals: - if id not in existing_inner_ids: - elem = etree.SubElement(compounddef, rf'inner{inner_type}') - elem.text = name - elem.set(r'refid', id) - elem.set(r'prot', r'public') # todo: this isn't necessarily correct - existing_inner_ids.add(id) - changed = True - if changed: - write_xml_to_file(xml, xml_file) - - # merge extracted implementations - if extracted_implementation: - for (hp, hfn, hid, impl) in implementation_header_data: - xml_file = Path(context.xml_dir, rf'{hid}.xml') - context.verbose(rf'Merging implementation nodes into {xml_file}') - xml = etree.parse(str(xml_file), parser=xml_parser) - compounddef = xml.getroot().find(r'compounddef') - changed = False - - innernamespaces = compounddef.findall(r'innernamespace') - for new_tag in implementation_header_innernamespaces[hid]: - matched = False - for existing_tag in innernamespaces: - if existing_tag.get(r'refid') == new_tag.get(r'refid'): - matched = True - break - if not matched: - compounddef.append(new_tag) - innernamespaces.append(new_tag) - changed = True - - sectiondefs = compounddef.findall(r'sectiondef') - for new_section in implementation_header_sectiondefs[hid]: - matched_section = False - for existing_section in sectiondefs: - if existing_section.get(r'kind') == new_section.get(r'kind'): - matched_section = True - - memberdefs = existing_section.findall(r'memberdef') - new_memberdefs = new_section.findall(r'memberdef') - for new_memberdef in new_memberdefs: - matched = False - for existing_memberdef in memberdefs: - if existing_memberdef.get(r'id') == new_memberdef.get(r'id'): - matched = True - break - - if not matched: - new_section.remove(new_memberdef) - existing_section.append(new_memberdef) - memberdefs.append(new_memberdef) - changed = True - break + if not matched: + compounddef.append(new_tag) + innernamespaces.append(new_tag) + changed = True + + sectiondefs = compounddef.findall(r'sectiondef') + for new_section in implementation_header_sectiondefs[hid]: + matched_section = False + for existing_section in sectiondefs: + if existing_section.get(r'kind') == new_section.get(r'kind'): + matched_section = True + + memberdefs = existing_section.findall(r'memberdef') + new_memberdefs = new_section.findall(r'memberdef') + for new_memberdef in new_memberdefs: + matched = False + for existing_memberdef in memberdefs: + if existing_memberdef.get(r'id') == new_memberdef.get(r'id'): + matched = True + break - if not matched_section: - compounddef.append(new_section) - sectiondefs.append(new_section) - changed = True - - if changed: - implementation_header_unused_keys.remove(hp) - write_xml_to_file(xml, xml_file) - - # sanity-check implementation header state - if implementation_header_unused_keys: - for key in implementation_header_unused_keys: - context.warning(rf"implementation_header: nothing extracted for '{key}'") - if implementation_header_unused_values: - for iid, idata in implementation_header_unused_values.items(): - context.warning(rf"implementation_header: nothing extracted from '{idata[0]}' for '{idata[1]}'") - - # delete the impl header xml files - if 1 and context.implementation_headers: - for hdata in implementation_header_data: - for (ip, ifn, iid) in hdata[3]: - delete_file(Path(context.xml_dir, rf'{iid}.xml'), logger=context.verbose_logger) - - # scan through the files and substitute impl header ids and paths as appropriate - if 1 and context.implementation_headers: - xml_files = get_all_files(context.xml_dir, any=('*.xml')) - for xml_file in xml_files: - context.verbose(rf"Re-linking implementation headers in '{xml_file}'") - xml_text = read_all_text_from_file(xml_file, logger=context.verbose_logger) - for (hp, hfn, hid, impl) in implementation_header_data: - for (ip, ifn, iid) in impl: - #xml_text = xml_text.replace(f'refid="{iid}"',f'refid="{hid}"') - xml_text = xml_text.replace(rf'compoundref="{iid}"', f'compoundref="{hid}"') - xml_text = xml_text.replace(ip, hp) - with BytesIO(bytes(xml_text, 'utf-8')) as b: - xml = etree.parse(b, parser=xml_parser) + if not matched: + new_section.remove(new_memberdef) + existing_section.append(new_memberdef) + memberdefs.append(new_memberdef) + changed = True + break + + if not matched_section: + compounddef.append(new_section) + sectiondefs.append(new_section) + changed = True + + if changed: + implementation_header_unused_keys.remove(hp) write_xml_to_file(xml, xml_file) + # sanity-check implementation header state + if implementation_header_unused_keys: + for key in implementation_header_unused_keys: + context.warning(rf"implementation_header: nothing extracted for '{key}'") + if implementation_header_unused_values: + for iid, idata in implementation_header_unused_values.items(): + context.warning(rf"implementation_header: nothing extracted from '{idata[0]}' for '{idata[1]}'") + + # delete the impl header xml files + if 1 and context.implementation_headers: + for hdata in implementation_header_data: + for (ip, ifn, iid) in hdata[3]: + delete_file(Path(context.xml_dir, rf'{iid}.xml'), logger=context.verbose_logger) + + # scan through the files and substitute impl header ids and paths as appropriate + if 1 and context.implementation_headers: + xml_files = get_all_files(context.xml_dir, any=('*.xml')) + for xml_file in xml_files: + context.verbose(rf"Re-linking implementation headers in '{xml_file}'") + xml_text = read_all_text_from_file(xml_file, logger=context.verbose_logger) + for (hp, hfn, hid, impl) in implementation_header_data: + for (ip, ifn, iid) in impl: + #xml_text = xml_text.replace(f'refid="{iid}"',f'refid="{hid}"') + xml_text = xml_text.replace(rf'compoundref="{iid}"', f'compoundref="{hid}"') + xml_text = xml_text.replace(ip, hp) + with BytesIO(bytes(xml_text, 'utf-8')) as b: + xml = etree.parse(b, parser=xml_parser) + write_xml_to_file(xml, xml_file) + + # convert the definition lists into compiled regexes since we've now extracted everything useful + context.code_blocks.namespaces = regex_or( + context.code_blocks.namespaces, pattern_prefix='(?:::)?', pattern_suffix='(?:::)?' + ) + context.code_blocks.types = regex_or(context.code_blocks.types, pattern_prefix='(?:::)?', pattern_suffix='(?:::)?') + context.code_blocks.enums = regex_or(context.code_blocks.enums, pattern_prefix='(?:::)?') + context.code_blocks.string_literals = regex_or(context.code_blocks.string_literals) + context.code_blocks.numeric_literals = regex_or(context.code_blocks.numeric_literals) + context.code_blocks.macros = regex_or(context.code_blocks.macros) + context.autolinks = tuple([(re.compile('(?\n' + # scripts + for script in context.scripts: + html_header += f'\n' + if context.theme != r'custom': + html_header += f'\n' + # metadata + def add_meta_kvp(key_name, key, content): + nonlocal html_header + html_header += f'\n' + + add_meta = lambda key, content: add_meta_kvp(r'name', key, content) + add_property = lambda key, content: add_meta_kvp(r'property', key, content) + add_itemprop = lambda key, content: add_meta_kvp(r'itemprop', key, content) + # metadata - project name + if context.name: + if r'twitter:title' not in context.meta_tags: + add_meta(r'twitter:title', context.name) + add_property(r'og:title', context.name) + add_itemprop(r'name', context.name) + # metadata - project author + if context.author: + if r'author' not in context.meta_tags: + add_meta(r'author', context.author) + add_property(r'article:author', context.author) + # metadata - project description + if context.description: + if r'description' not in context.meta_tags: + add_meta(r'description', context.description) + if r'twitter:description' not in context.meta_tags: + add_meta(r'twitter:description', context.description) + add_property(r'og:description', context.description) + add_itemprop(r'description', context.description) + # metadata - robots + if not context.robots: + if r'robots' not in context.meta_tags: + add_meta(r'robots', r'noindex, nofollow') + if r'googlebot' not in context.meta_tags: + add_meta(r'googlebot', r'noindex, nofollow') + # metadata - misc + if r'format-detection' not in context.meta_tags: + add_meta(r'format-detection', r'telephone=no') + if r'generator' not in context.meta_tags: + add_meta(r'generator', rf'Poxy v{context.version_string}') + if r'referrer' not in context.meta_tags: + add_meta(r'referrer', r'strict-origin-when-cross-origin') + # metadata - additional user-specified tags + for name, content in context.meta_tags.items(): + add_meta(name, content) + # html_header + if context.html_header: + html_header += f'{context.html_header}\n' + html_header = html_header.rstrip() + + # build + write conf.py + with StringIO(newline='\n') as conf_py: + conf = lambda s='', end='\n': print(reindent(s, indent=''), file=conf_py, end=end) + + # basic properties + conf(rf"DOXYFILE = r'{context.doxyfile_path}'") + conf(r"STYLESHEETS = []") # suppress the default behaviour + conf(rf'HTML_HEADER = """{html_header}"""') + if context.theme == r'dark': + conf(r"THEME_COLOR = '#22272e'") + elif context.theme == r'light': + conf(r"THEME_COLOR = '#cb4b16'") + if context.favicon: + conf(rf"FAVICON = r'{context.favicon}'") + elif context.theme == r'dark': + conf(rf"FAVICON = 'favicon-dark.png'") + elif context.theme == r'light': + conf(rf"FAVICON = 'favicon-light.png'") + conf(rf'SHOW_UNDOCUMENTED = {context.sources.extract_all}') + conf(r'CLASS_INDEX_EXPAND_LEVELS = 3') + conf(r'FILE_INDEX_EXPAND_LEVELS = 3') + conf(r'CLASS_INDEX_EXPAND_INNER = True') + conf(r'SEARCH_DOWNLOAD_BINARY = False') + conf(r'SEARCH_DISABLED = False') + + # navbar + NAVBAR_ALIASES = { + # poxy -> doxygen + r'classes': r'annotated', + r'groups': r'modules' + } + NAVBAR_TO_KIND = { + r'annotated': (r'class', r'struct', r'union'), + r'concepts': (r'concept', ), + r'namespaces': (r'namespace', ), + r'pages': (r'page', ), + r'modules': (r'group', ), + r'files': (r'file', r'dir') + } + navbar = ([], []) + if context.navbar: + # populate the navbar + bar = [(NAVBAR_ALIASES[b] if b in NAVBAR_ALIASES else b) for b in context.navbar] + # remove links to index pages that will have no entries + for i in range(len(bar)): + if bar[i] not in NAVBAR_TO_KIND: + continue + found = False + for kind in NAVBAR_TO_KIND[bar[i]]: + if kind in context.compound_kinds: + found = True + break + if not found: + bar[i] = None + bar = [b for b in bar if b is not None] + # handle theme and repo links + for i in range(len(bar)): + if bar[i] == r'repo' and context.repo: + icon_path = Path(dirs.DATA, context.repo.icon_filename) + if icon_path.exists(): + svg = SVG(icon_path, logger=context.verbose_logger, root_id=r'poxy-repo-icon') + bar[i] = ( + rf'{svg}', [] + ) + else: + bar[i] = None + elif bar[i] == r'theme': + svg = SVG( + Path(dirs.DATA, r'poxy-icon-theme.svg'), + logger=context.verbose_logger, + root_id=r'poxy-theme-switch-img' + ) + bar[i] = ( + r'{svg}', [] + ) + bar = [b for b in bar if b is not None] + # automatically overflow onto the second row + split = min(max(int(len(bar) / 2) + len(bar) % 2, 2), len(bar)) + for b, i in ((bar[:split], 0), (bar[split:], 1)): + for j in range(len(b)): + if isinstance(b[j], tuple): + navbar[i].append(b[j]) + else: + navbar[i].append((None, b[j], [])) + for i in (0, 1): + if navbar[i]: + conf(f'LINKS_NAVBAR{i+1} = [\n\t', end='') + conf(',\n\t'.join([rf'{b}' for b in navbar[i]])) + conf(r']') + else: + conf(rf'LINKS_NAVBAR{i+1} = []') + + # footer + conf(r"FINE_PRINT = r'''") + footer = [] + if context.repo: + footer.append(rf'{type(context.repo).__name__}') + footer.append(rf'Report an issue') + if context.changelog: + footer.append(rf'Changelog') + if context.license and context.license[r'uri']: + footer.append(rf'License') + if context.generate_tagfile: + footer.append( + rf'Doxygen tagfile' + ) + if footer: + for i in range(1, len(footer)): + footer[i] = r' • ' + footer[i] + footer.append(r'

') + footer.append(r'Site generated using Poxy') + for i in range(len(footer)): + conf(rf" {footer[i]}") + conf(r"'''") + + conf_py_text = conf_py.getvalue() + context.verbose(r'm.css conf.py:') + context.verbose(conf_py_text, indent=r' ') + + # write conf.py + context.verbose(rf'Writing {context.mcss_conf_path}') + with open(context.mcss_conf_path, r'w', encoding=r'utf-8', newline='\n') as f: + f.write(conf_py_text) + _worker_context = None @@ -997,7 +1077,7 @@ def _initialize_worker(context): -def postprocess_html_file(path, context=None): +def postprocess_html_file(path, context: Context = None): assert path is not None assert isinstance(path, Path) assert path.is_absolute() @@ -1007,25 +1087,51 @@ def postprocess_html_file(path, context=None): global _worker_context context = _worker_context assert context is not None - assert isinstance(context, project.Context) + assert isinstance(context, Context) context.info(rf'Post-processing {path}') - changed = False + text = None + html = None + + def switch_to_html(): + nonlocal context + nonlocal text + nonlocal html + if html is not None: + return + html = soup.HTMLDocument(text, logger=context.verbose_logger) + + def switch_to_text(): + nonlocal context + nonlocal text + nonlocal html + if html is None: + return + html.smooth() + text = str(html) + html = None + try: + text = read_all_text_from_file(path, logger=context.verbose_logger) + changed = False + for fix in context.fixers: if isinstance(fix, fixers.HTMLFixer): - doc = soup.HTMLDocument(path, logger=context.verbose_logger) - if fix(doc, context): - doc.smooth() - doc.flush() + switch_to_html() + if fix(context, html, path): changed = True + html.smooth() elif isinstance(fix, fixers.PlainTextFixer): - doc = [read_all_text_from_file(path, logger=context.verbose_logger), path] - if fix(doc, context): - context.verbose(rf'Writing {path}') - with open(path, 'w', encoding='utf-8', newline='\n') as f: - f.write(doc[0]) - changed = True + switch_to_text() + prev_text = text + text = fix(context, prev_text, path) + changed = changed or prev_text != text + + if changed: + switch_to_text() + context.verbose(rf'Writing {path}') + with open(path, 'w', encoding='utf-8', newline='\n') as f: + f.write(text) except Exception as e: context.info(rf'{type(e).__name__} raised while post-processing {path}') @@ -1034,13 +1140,11 @@ def postprocess_html_file(path, context=None): context.info(rf'Error occurred while post-processing {path}') raise - return changed - -def postprocess_html(context): +def postprocess_html(context: Context): assert context is not None - assert isinstance(context, project.Context) + assert isinstance(context, Context) files = filter_filenames( get_all_files(context.html_dir, any=('*.html', '*.htm')), context.html_include, context.html_exclude @@ -1048,44 +1152,43 @@ def postprocess_html(context): if not files: return - threads = min(len(files), context.threads, 8) # diminishing returns after 8 - - with ScopeTimer(rf'Post-processing {len(files)} HTML files', print_start=True, print_end=context.verbose_logger): - context.fixers = ( - fixers.MarkTOC(), - fixers.CodeBlocks(), - fixers.Banner(), - fixers.CPPModifiers1(), - fixers.CPPModifiers2(), - fixers.CPPTemplateTemplate(), - fixers.StripIncludes(), - fixers.AutoDocLinks(), - fixers.Links(), - fixers.CustomTags(), - fixers.EmptyTags(), - fixers.ImplementationDetails(), - fixers.MarkdownPages(), - fixers.InjectSVGs(), - ) - context.verbose(rf'Post-processing {len(files)} HTML files...') - if threads > 1: - with futures.ProcessPoolExecutor( - max_workers=threads, initializer=_initialize_worker, initargs=(context, ) - ) as executor: - jobs = [executor.submit(postprocess_html_file, file) for file in files] - for future in futures.as_completed(jobs): + context.fixers = ( + fixers.MarkTOC(), + fixers.CodeBlocks(), + fixers.Banner(), + fixers.CPPModifiers1(), + fixers.CPPModifiers2(), + fixers.CPPTemplateTemplate(), + fixers.StripIncludes(), + fixers.AutoDocLinks(), + fixers.Links(), + fixers.CustomTags(), + fixers.EmptyTags(), + fixers.ImplementationDetails(), + fixers.MarkdownPages(), + fixers.InjectSVGs(), + ) + + threads = min(len(files), context.threads, 16) + context.info(rf'Post-processing {len(files)} HTML files on {threads} thread{"s" if threads > 1 else ""}...') + if threads > 1: + with futures.ProcessPoolExecutor( + max_workers=threads, initializer=_initialize_worker, initargs=(context, ) + ) as executor: + jobs = [executor.submit(postprocess_html_file, file) for file in files] + for future in futures.as_completed(jobs): + try: + future.result() + except: try: - future.result() - except: - try: - executor.shutdown(wait=False, cancel_futures=True) - except TypeError: - executor.shutdown(wait=False) - raise + executor.shutdown(wait=False, cancel_futures=True) + except TypeError: + executor.shutdown(wait=False) + raise - else: - for file in files: - postprocess_html_file(file, context) + else: + for file in files: + postprocess_html_file(file, context) @@ -1169,6 +1272,69 @@ def extract_warnings(outputs): +def run_doxygen(context: Context): + assert context is not None + assert isinstance(context, Context) + with make_temp_file() as stdout, make_temp_file() as stderr: + try: + subprocess.run([str(context.doxygen_path), str(context.doxyfile_path)], + check=True, + stdout=stdout, + stderr=stderr, + cwd=context.input_dir) + except: + context.info(r'Doxygen failed!') + dump_output_streams(context, read_output_streams(stdout, stderr), source=r'Doxygen') + raise + if context.is_verbose() or context.warnings.enabled: + outputs = read_output_streams(stdout, stderr) + if context.is_verbose(): + dump_output_streams(context, outputs, source=r'Doxygen') + if context.warnings.enabled: + warnings = extract_warnings(outputs) + for w in warnings: + context.warning(w) + + # remove the local paths from the tagfile since they're meaningless (and a privacy breach) + if context.tagfile_path: + text = read_all_text_from_file(context.tagfile_path, logger=context.verbose_logger) + text = re.sub(r'\n\s*?.+?\s*?\n', '\n', text, re.S) + context.verbose(rf'Writing {context.tagfile_path}') + with open(context.tagfile_path, 'w', encoding='utf-8', newline='\n') as f: + f.write(text) + + + +def run_mcss(context: Context): + assert context is not None + assert isinstance(context, Context) + with make_temp_file() as stdout, make_temp_file() as stderr: + doxy_args = [str(context.mcss_conf_path), r'--no-doxygen', r'--sort-globbed-files'] + if context.is_verbose(): + doxy_args.append(r'--debug') + try: + run_python_script( + Path(dirs.MCSS, r'documentation/doxygen.py'), + *doxy_args, + stdout=stdout, + stderr=stderr, + cwd=context.input_dir + ) + except: + context.info(r'm.css failed!') + dump_output_streams(context, read_output_streams(stdout, stderr), source=r'm.css') + raise + if context.is_verbose() or context.warnings.enabled: + outputs = read_output_streams(stdout, stderr) + if context.is_verbose(): + dump_output_streams(context, outputs, source=r'm.css') + if context.warnings.enabled: + warnings = extract_warnings(outputs) + for w in warnings: + context.warning(w) + + + def run( config_path=None, output_dir='.', @@ -1177,7 +1343,6 @@ def run( verbose=False, doxygen_path=None, logger=None, - dry_run=False, xml_only=False, html_include=None, html_exclude=None, @@ -1186,7 +1351,9 @@ def run( copy_assets=True ): - with project.Context( + timer = lambda desc: ScopeTimer(desc, print_start=True, print_end=context.verbose_logger) + + with Context( config_path=config_path, output_dir=output_dir, threads=threads, @@ -1194,7 +1361,6 @@ def run( verbose=verbose, doxygen_path=doxygen_path, logger=logger, - dry_run=dry_run, xml_only=xml_only, html_include=html_include, html_exclude=html_exclude, @@ -1203,137 +1369,22 @@ def run( copy_assets=copy_assets ) as context: - # preprocess the doxyfile preprocess_doxyfile(context) - context.verbose_object(r'Context.warnings', context.warnings) - - if context.dry_run: - return - - # resolve any uri tagfiles - if context.unresolved_tagfiles: - with ScopeTimer(r'Resolving remote tagfiles', print_start=True, print_end=context.verbose_logger) as t: - for source, (file, _) in context.tagfiles.items(): - if file.exists() or not is_uri(source): - continue - context.verbose(rf'Downloading {source} => {file}') - response = requests.get(source, allow_redirects=True, stream=False, timeout=30) - context.verbose(rf'Writing {file}') - with open(file, 'w', encoding='utf-8', newline='\n') as f: - f.write(response.text) - - make_temp_file = lambda: tempfile.SpooledTemporaryFile(mode='w+', newline='\n', encoding='utf-8') + preprocess_tagfiles(context) + preprocess_changelog(context) - # precondition the change log page (at this point it is already a temp copy) - if context.changelog: - text = read_all_text_from_file(context.changelog, logger=context.verbose_logger).strip() - text = text.replace('\r\n', '\n') - text = re.sub(r'\n\n', r'', text) - if context.repo: - text = re.sub(r'#([0-9]+)', lambda m: rf'[#{m[1]}]({context.repo.make_issue_uri(m[1])})', text) - text = re.sub(r'!([0-9]+)', lambda m: rf'[!{m[1]}]({context.repo.make_pull_request_uri(m[1])})', text) - text = re.sub(r'@([a-zA-Z0-9_-]+)', lambda m: rf'[@{m[1]}]({context.repo.make_user_uri(m[1])})', text) - text = text.replace(r'&', r'__poxy_thiswasan_amp') - text = text.replace(r'️', r'__poxy_thiswasan_fe0f') - text = text.replace(r'@', r'__poxy_thiswasan_at') - if text.find(r'@tableofcontents') == -1 and text.find('\\tableofcontents' - ) == -1 and text.find(r'[TOC]') == -1: - #text = f'[TOC]\n\n{text}' - nlnl = text.find(r'\n\n') - if nlnl != -1: - text = f'{text[:nlnl]}\n\n\\tableofcontents\n\n{text[nlnl:]}' - pass - text += '\n\n' - with open(context.changelog, r'w', encoding=r'utf-8', newline='\n') as f: - f.write(text) - - # run doxygen to generate the xml - if 1: - with ScopeTimer( - r'Generating XML files with Doxygen', print_start=True, print_end=context.verbose_logger - ) as t: - with make_temp_file() as stdout, make_temp_file() as stderr: - try: - subprocess.run([str(context.doxygen_path), - str(context.doxyfile_path)], - check=True, - stdout=stdout, - stderr=stderr, - cwd=context.input_dir) - except: - context.info(r'Doxygen failed!') - dump_output_streams(context, read_output_streams(stdout, stderr), source=r'Doxygen') - raise - if context.is_verbose() or context.warnings.enabled: - outputs = read_output_streams(stdout, stderr) - if context.is_verbose(): - dump_output_streams(context, outputs, source=r'Doxygen') - if context.warnings.enabled: - warnings = extract_warnings(outputs) - for w in warnings: - context.warning(w) - - # remove the local paths from the tagfile since they're meaningless (and a privacy breach) - if context.tagfile_path is not None and context.tagfile_path.exists(): - text = read_all_text_from_file(context.tagfile_path, logger=context.verbose_logger) - text = re.sub(r'\n\s*?.+?\s*?\n', '\n', text, re.S) - context.verbose(rf'Writing {context.tagfile_path}') - with open(context.tagfile_path, 'w', encoding='utf-8', newline='\n') as f: - f.write(text) - - # post-process xml files - if 1: + # generate + postprocess XML + with timer(r'Generating XML files with Doxygen') as t: + run_doxygen(context) + with timer(r'Post-processing XML files') as t: postprocess_xml(context) - if context.xml_only: return - context.verbose_object(r'Context.code_blocks', context.code_blocks) - - # compile regexes - # (done here because doxygen and xml preprocessing adds additional values to these lists) - context.code_blocks.namespaces = regex_or( - context.code_blocks.namespaces, pattern_prefix='(?:::)?', pattern_suffix='(?:::)?' - ) - context.code_blocks.types = regex_or( - context.code_blocks.types, pattern_prefix='(?:::)?', pattern_suffix='(?:::)?' - ) - context.code_blocks.enums = regex_or(context.code_blocks.enums, pattern_prefix='(?:::)?') - context.code_blocks.string_literals = regex_or(context.code_blocks.string_literals) - context.code_blocks.numeric_literals = regex_or(context.code_blocks.numeric_literals) - context.code_blocks.macros = regex_or(context.code_blocks.macros) - context.autolinks = tuple([(re.compile('(? str: + return str(self.__doc) def new_tag(self, tag_name, parent=None, string=None, class_=None, index=None, before=None, after=None, **kwargs): tag = self.__doc.new_tag(tag_name, **kwargs) diff --git a/poxy/svg.py b/poxy/svg.py index fed7de6..31f883f 100644 --- a/poxy/svg.py +++ b/poxy/svg.py @@ -76,4 +76,4 @@ def __init__( del attrs[r'class'] def __str__(self) -> str: - return etree.tostring(self.__xml, encoding=r'unicode', xml_declaration=False, pretty_print=True) + return etree.tostring(self.__xml, encoding=r'unicode', xml_declaration=False, pretty_print=False) diff --git a/poxy/utils.py b/poxy/utils.py index cf97c15..0d7bdad 100644 --- a/poxy/utils.py +++ b/poxy/utils.py @@ -92,7 +92,7 @@ def filter_filenames(files, include, exclude): def download_text(uri: str, timeout=10, encoding='utf-8') -> str: assert uri is not None global DOWNLOAD_HEADERS - response = requests.get(str(uri), headers=DOWNLOAD_HEADERS, timeout=timeout) + response = requests.get(str(uri), headers=DOWNLOAD_HEADERS, timeout=timeout, stream=False, allow_redirects=True) if encoding is not None: response.encoding = encoding return response.text @@ -102,7 +102,7 @@ def download_text(uri: str, timeout=10, encoding='utf-8') -> str: def download_binary(uri: str, timeout=10) -> bytes: assert uri is not None global DOWNLOAD_HEADERS - response = requests.get(str(uri), headers=DOWNLOAD_HEADERS, timeout=timeout) + response = requests.get(str(uri), headers=DOWNLOAD_HEADERS, timeout=timeout, stream=False, allow_redirects=True) return response.content diff --git a/tests/regenerate_tests.py b/tests/regenerate_tests.py index 0e6969a..97c675c 100644 --- a/tests/regenerate_tests.py +++ b/tests/regenerate_tests.py @@ -4,6 +4,7 @@ # See https://github.com/marzer/poxy/blob/master/LICENSE for the full license text. # SPDX-License-Identifier: MIT +import re from utils import * from pathlib import Path @@ -27,19 +28,30 @@ def regenerate_expected_outputs(): print(rf"Regenerating {subdir}...") run_poxy(subdir, r'--nocleanup', r'--noassets') - GARBAGE = (r'*.xslt', r'*.xsd', r'Doxyfile.xml') + # delete junk + GARBAGE = (r'*.xslt', r'*.xsd', r'Doxyfile.xml', r'*.tagfile.xml') garbage = enumerate_files(html_dir, any=GARBAGE) garbage += enumerate_files(xml_dir, any=GARBAGE) for file in garbage: delete_file(file, logger=True) - for file_path in enumerate_files(html_dir, any=(r'*.html')): - html = read_all_text_from_file(file_path) - html = html.replace(r'href="poxy/poxy.css"', r'href="../../../poxy/data/css/poxy.css"') - html = html.replace(r'src="poxy/poxy.js"', r'src="../../../poxy/data/poxy.js"') - print(rf"Writing {file_path}") - with open(file_path, r'w', encoding=r'utf-8', newline='\n') as f: - f.write(html) + # normalize html files + for path in enumerate_files(html_dir, any=(r'*.html')): + text = read_all_text_from_file(path) + text = text.replace(r'href="poxy/poxy.css"', r'href="../../../poxy/data/css/poxy.css"') + text = text.replace(r'src="poxy/poxy.js"', r'src="../../../poxy/data/poxy.js"') + text = re.sub(r'Poxy v[0-9]+[.][0-9]+[.][0-9]+', r'Poxy v0.0.0', text) + print(rf"Writing {path}") + with open(path, r'w', encoding=r'utf-8', newline='\n') as f: + f.write(text) + + # normalize xml files + for path in enumerate_files(xml_dir, any=(r'*.xml')): + text = read_all_text_from_file(path) + text = re.sub(r'version="\s*[0-9]+[.][0-9]+[.][0-9]+\s*"', r'version="0.0.0"', text) + print(rf"Writing {path}") + with open(path, r'w', encoding=r'utf-8', newline='\n') as f: + f.write(text) html_dir.rename(expected_html_dir) xml_dir.rename(expected_xml_dir) diff --git a/tests/test_empty_project/expected_html/annotated.html b/tests/test_empty_project/expected_html/annotated.html index 5b13b10..5c69899 100644 --- a/tests/test_empty_project/expected_html/annotated.html +++ b/tests/test_empty_project/expected_html/annotated.html @@ -9,7 +9,7 @@ - + @@ -26,26 +26,10 @@