compat: fix incompatibilities and bugs with older sphinx versions

BTW: add pylint *disables* for handled *deprecation* warnings

Bugfixes:

- Sphinx <= 1.4 (bug in docutils >= 0.13) : HTML Builders crashes with
  docutils-0.13 `[ref] <https://github.com/sphinx-doc/sphinx/pull/3217>`__

Downward compatibility:

- Sphinx >= 1.6: ``Sphinx.warn()``, ``Sphinx.info()`` and other logging methods are
  now deprecated.  Please use ``sphinx.util.logging`` instead.  It will be
  removed in Sphinx-2.0.

- Sphinx >= 1.7: ``sphinx.ext.autodoc.AutodocReporter`` is replaced by
  ``sphinx.util.docutils.  switch_source_input()`` and now deprecated.  It will
  be removed in Sphinx-2.0.

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>

linuxdoc/compat.py

@@ -1,30 +1,77 @@
 # -*- coding: utf-8; mode: python -*-
-# pylint: disable=invalid-name, missing-docstring
-u"""compat
-    ~~~~~~
+# pylint: disable=C, unused-import, invalid-name, missing-docstring
+u"""
+compat
+~~~~~~
 
-    Implementation of a compatibility layer for sphinx related modules.
+Implementation of a compatibility layer for sphinx and docutils related modules.
 
-    :copyright:  Copyright (C) 2018 Markus Heiser
-    :license:    GPL Version 2, June 1991 see Linux/COPYING for details.
+:copyright:  Copyright (C) 2018 Markus Heiser
+:license:    GPL Version 2, June 1991 see Linux/COPYING for details.
 
-    Downward compatibility is unfortunately no strength of sphinx-doc and patch
-    levels not really exists or if so they are not shipped within LTS
-    distributions.  Therefor a minimal compatibility *layer* is needed.
+Downward compatibility is unfortunately no strength of sphinx-doc `[ref]
+<https://github.com/sphinx-doc/sphinx/issues/3212#issuecomment-283756374>`__ and
+patch levels are not really exists or if so they are not shipped within LTS
+distributions.  Therefor a minimal *compatibility layer* is needed.  Even if we
+do our best here, there are also a lot of incompatibilities in between sphinx
+docutils whose fixing is out of the scope of this linuxdoc project `[ref]
+<https://www.kernel.org/doc/html/latest/doc-guide/sphinx.html#sphinx-install>`__.
+
+To get best results (and less warnings) its inevitable to use latest sphinx-doc
+version and the RTD in a python3 `virtualenv <https://virtualenv.pypa.io>`__::
+
+  $ virtualenv3 py3
+  $ . py3/bin/activate
+  $ pip install sphinx sphinx_rtd_theme
+
+The following incompatibilities will be handled by this layer.  More details see
+Sphinx-doc’s `CHANGELOG <http://www.sphinx-doc.org/en/master/changes.html>`__
+and docutils `RELEASE-NOTES
+<http://docutils.sourceforge.net/RELEASE-NOTES.html>`__
+
+Bugfixes:
+
+- Sphinx <= 1.4 (bug in docutils >= 0.13) : HTML Builders crashes with
+  docutils-0.13 `[ref] <https://github.com/sphinx-doc/sphinx/pull/3217>`__
+
+Downward compatibility:
+
+- Sphinx >= 1.6: ``Sphinx.warn()``, ``Sphinx.info()`` and other logging methods are
+  now deprecated.  Please use ``sphinx.util.logging`` instead.  It will be
+  removed in Sphinx-2.0.
+
+- Sphinx >= 1.7: ``sphinx.ext.autodoc.AutodocReporter`` is replaced by
+  ``sphinx.util.docutils.  switch_source_input()`` and now deprecated.  It will
+  be removed in Sphinx-2.0.
 
 """
 
+import docutils
 import sphinx
 
 # Get Sphinx version
 major, minor, patch = sphinx.version_info[:3]  # pylint: disable=invalid-name
+docutils_major, docutils_minor, docutils_patch = docutils.__version_info__[:3]
+
 
-if major == 1 and minor > 5:
-    # new logging format started with sphinx 1.6
+if (major == 1 and minor <= 4) and (docutils_major == 0 and docutils_minor>=13):
+    # to fix the docutils bug, we need a ugly hack, extending the code
+    # object of sphinx's HTMLTranslator.depart_image method
+    from sphinx.writers import html
+    _origin_HTMLTranslator_depart_image = html.HTMLTranslator.depart_image
+    def _wrap_HTMLTranslator_depart_image(__self, node):
+        _origin_HTMLTranslator_depart_image(__self, node)
+        if node['uri'].lower().endswith(('svg', 'svgz')):
+            __self.context.pop()
+    html.HTMLTranslator.depart_image = _wrap_HTMLTranslator_depart_image
+
+
+if major >= 1 and minor >= 6:
     from sphinx.util import logging
     getLogger = logging.getLogger
 
 else:
+    # workaround for Sphinx < 1.6
     import logging
     from collections import defaultdict
     VERBOSE=15
@@ -87,3 +134,29 @@ def getLogger(name):
 
     logger = logging.getLogger('sphinx')
     logger.setLevel(logging.DEBUG)
+
+if major >= 1 and minor >= 7:
+    from sphinx.util.docutils import switch_source_input
+
+else:
+    # workaround for Sphinx < 1.7
+    from docutils.statemachine import StateMachine
+    from contextlib import contextmanager
+
+    @contextmanager
+    def switch_source_input(state, content):
+        # type: (State, ViewList) -> Generator[None, None, None]
+        """Switch current source input of state temporarily."""
+        try:
+            # remember the original ``get_source_and_line()`` method
+            get_source_and_line = state.memo.reporter.get_source_and_line
+
+            # replace it by new one
+            state_machine = StateMachine([], None)
+            state_machine.input_lines = content
+            state.memo.reporter.get_source_and_line = state_machine.get_source_and_line
+
+            yield
+        finally:
+            # restore the method
+            state.memo.reporter.get_source_and_line = get_source_and_line

linuxdoc/kfigure.py

@@ -142,6 +142,8 @@ def setupTools(app):
     This function is called once, when the builder is initiated.
     """
     global dot_cmd, convert_cmd  # pylint: disable=global-statement
+
+    # pylint: disable=deprecated-method
     app_log.verbose("kfigure: check installed tools ...")
 
     dot_cmd = which('dot')
@@ -276,6 +278,7 @@ def dot2format(app, dot_fname, out_fname):
     with open(out_fname, "w") as out:
         exit_code = subprocess.call(cmd, stdout = out)
         if exit_code != 0:
+            # pylint: disable=deprecated-method
             app_log.warn("Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
     return bool(exit_code == 0)
 
@@ -293,6 +296,7 @@ def svg2pdf(app, svg_fname, pdf_fname):
     # use stdout and stderr from parent
     exit_code = subprocess.call(cmd)
     if exit_code != 0:
+        # pylint: disable=deprecated-method
         app_log.warn("Error #%d when calling: %s" % (exit_code, " ".join(cmd)))
     return bool(exit_code == 0)
 
@@ -386,6 +390,7 @@ def visit_kernel_render(self, node):
     app = self.builder.app
     srclang = node.get('srclang')
 
+    # pylint: disable=deprecated-method
     app_log.verbose('visit kernel-render node lang: "%s"' % (srclang))
 
     tmp_ext = RENDER_MARKUP_EXT.get(srclang, None)

linuxdoc/rstKernelDoc.py

@@ -29,8 +29,6 @@
 from docutils.utils import SystemMessage
 from docutils.statemachine import ViewList
 
-from sphinx.ext.autodoc import AutodocReporter
-
 from . import compat
 from . import kernel_doc as kerneldoc
 
@@ -69,6 +67,8 @@ def setup(app):
 class KernelDocParser(kerneldoc.Parser):
 # ==============================================================================
 
+    # pylint: disable=deprecated-method
+
     def __init__(self, app, *args, **kwargs):
         super(KernelDocParser, self).__init__(*args, **kwargs)
         self.app = app
@@ -337,7 +337,6 @@ def getNodes(self):  # pylint: disable=too-many-branches, too-many-statements, t
         translator = kerneldoc.ReSTTranslator()
         lines      = ""
         content    = WriterList(self.parser)
-        node       = nodes.section()
 
         # translate
 
@@ -402,14 +401,17 @@ def getNodes(self):  # pylint: disable=too-many-branches, too-many-statements, t
             for l in lines.split("\n"):
                 content.append(l, reSTfname, self.lineno)
 
-        buf = self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter
-        self.state.memo.reporter = AutodocReporter(content, self.state.memo.reporter)
-        self.state.memo.title_styles, self.state.memo.section_level = [], 0
-        try:
-            self.state.nested_parse(content, 0, node, match_titles=1)
-        finally:
-            self.state.memo.title_styles, self.state.memo.section_level, self.state.memo.reporter = buf
-
+        node = nodes.section()
+        # necessary so that the child nodes get the right source/line set
+        node.document = self.state.document
+        with compat.switch_source_input(self.state, content):
+            # hack around title style bookkeeping
+            buf = self.state.memo.title_styles, self.state.memo.section_level
+            self.state.memo.title_styles, self.state.memo.section_level = [], 0
+            try:
+                self.state.nested_parse(content, 0, node, match_titles=1)
+            finally:
+                self.state.memo.title_styles, self.state.memo.section_level = buf
         return node.children