Adding source files for docs.delta.io

docs/generate_api_docs.py → docs/apis/generate_api_docs.py

@@ -27,7 +27,7 @@ def main():
     verbose = args.verbose
 
     # Set up the directories
-    docs_root_dir = os.path.dirname(os.path.realpath(__file__))
+    docs_root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
     repo_root_dir = os.path.dirname(docs_root_dir)
 
     # --- dirs where docs are generated

docs/generate_docs.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import subprocess
+import random
+import shutil
+import string
+import tempfile
+
+
+def main():
+    """Script to manage the deployment of Delta Lake docs to the hosting bucket.
+       To build the docs:
+       $ generate_docs --livehtml
+
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--livehtml",
+        action="store_true",
+        help="Build and serve a local build of docs")
+    parser.add_argument(
+        "--api-docs",
+        action="store_true",
+        help="Generate the API docs")
+
+    args = parser.parse_args()
+
+    docs_root_dir = os.path.dirname(os.path.realpath(__file__))
+    api_docs_root_dir = os.path.join(docs_root_dir, "apis")
+
+    with WorkingDirectory(docs_root_dir):
+        html_output = os.path.join(docs_root_dir, '_site', 'html')
+        html_source = os.path.join(docs_root_dir, 'source')
+        print("Building content")
+        env = { "TARGET_CLOUD": "delta-oss-only" }
+
+        sphinx_cmd = "sphinx-build"
+        if args.livehtml:
+            sphinx_cmd = "sphinx-autobuild"
+        build_docs_args = "%s -b html -d /tmp/build/doctrees %s %s" % (
+            sphinx_cmd, html_source, html_output)
+        if args.api_docs:
+            copy_api_docs(args.api_docs_location, html_output)
+        run_cmd(build_docs_args, env=env, shell=True, stream_output=True)
+
+
+def copy_api_docs(apl_docs_dir, target_loc):
+    print("Building API docs")
+    with WorkingDirectory(apl_docs_dir):
+        run_cmd(["python generate_api_docs.py"], shell=True, stream_output=True)
+        assert os.path.exists(os.path.join(apl_docs_dir, "_site", "api")), \
+            "Doc generation didn't create the expected api directory"
+        shutil.copytree(os.path.join(apl_docs_dir, "_site", "api"), os.path.join(target_loc, "api"))
+
+
+class WorkingDirectory(object):
+    def __init__(self, working_directory):
+        self.working_directory = working_directory
+        self.old_workdir = os.getcwd()
+
+    def __enter__(self):
+        os.chdir(self.working_directory)
+
+    def __exit__(self, type, value, traceback):
+        os.chdir(self.old_workdir)
+
+
+def run_cmd(cmd, throw_on_error=True, env=None, stream_output=False, **kwargs):
+    """Runs a command as a child process.
+
+    A convenience wrapper for running a command from a Python script.
+    Keyword arguments:
+    cmd -- the command to run, as a list of strings
+    throw_on_error -- if true, raises an Exception if the exit code of the program is nonzero
+    env -- additional environment variables to be defined when running the child process
+    stream_output -- if true, does not capture standard output and error; if false, captures these
+      streams and returns them
+
+    Note on the return value: If stream_output is true, then only the exit code is returned. If
+    stream_output is false, then a tuple of the exit code, standard output and standard error is
+    returned.
+    """
+    cmd_env = os.environ.copy()
+    if env:
+        cmd_env.update(env)
+
+    if stream_output:
+        child = subprocess.Popen(cmd, env=cmd_env, **kwargs)
+        exit_code = child.wait()
+        if throw_on_error and exit_code != 0:
+            raise Exception("Non-zero exitcode: %s" % exit_code)
+        return exit_code
+    else:
+        child = subprocess.Popen(
+            cmd,
+            env=cmd_env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            **kwargs)
+        (stdout, stderr) = child.communicate()
+        exit_code = child.wait()
+        if throw_on_error and exit_code != 0:
+            raise Exception(
+                "Non-zero exitcode: %s\n\nSTDOUT:\n%s\n\nSTDERR:%s" %
+                (exit_code, stdout, stderr))
+        return exit_code, stdout.decode("utf-8"), stderr.decode("utf-8")
+
+
+if __name__ == "__main__":
+    main()

docs/shared/extensions/code_language_tabs/README.md

@@ -0,0 +1,81 @@
+# Code language tabs
+
+The module adds a new directive, `code-language-tabs` to [Sphinx](http://www.sphinx-doc.org/).
+
+Tested with Sphinx 1.8.5.
+
+## Usage
+
+### conf.py
+
+```python
+extensions = ['code_language_tabs']
+```
+
+### docs source
+
+#### short form
+
+```markdown
+.. code-language-tabs::
+
+    ```python
+    python code
+    ```
+
+    ```java
+    java code
+    ```
+
+    ```dotnet
+    dotnet code
+    ```
+
+```
+
+#### mixed form
+
+``` markdown
+.. code-language-tabs::
+
+    .. lang:: python
+
+        some arbitrary python related content
+
+    .. lang:: java
+
+        some arbitrary java related content
+
+    .. lang:: dotnet
+
+        some arbitrary .net related content
+```
+
+## Notes
+
+* In the `short form` the directive may contain only `literal blocks`. If anything else faced inside the body - an exception would be thrown.
+* In the `mixed form` the directive may contain only `lang` children, which represent particular pages. The children themselves may contain any arbitrary content.
+* You can not mix `short form` and `mixed form` into a single directive body.
+* The blocks order in final HTML is the same as in sources. The directive does not reorder blocks.
+
+## Output mode
+
+The extension might require to function differently when building different outputs. So there is a config setting for changing the extension output mode.
+
+### code-language-tabs-mode
+
+Type: `str`
+
+Default: `html`
+
+Options:
+
+* `html` - produce an output for further handling
+* `markdown` - produce an output compatible with Microsoft docs
+* `off` - outputs the directive content directly, as there was no directive used
+
+To pass a config value through the command line, see the [command-line options](https://www.sphinx-doc.org/en/1.5/man/sphinx-build.html):
+
+```bash
+sphinx-build -D code-language-tabs-mode=markdown <other_options>
+```

docs/shared/extensions/code_language_tabs/__init__.py

@@ -0,0 +1,13 @@
+"""Code language tabs initialization."""
+from sphinx.application import Sphinx
+from .directives import CodeLanguageTabs, Lang
+
+
+def setup(app: Sphinx):
+    """Set up the required directives and configs.
+
+    :param Sphinx app: The current sphinx app.
+    """
+    app.add_directive('code-language-tabs', CodeLanguageTabs)
+    app.add_directive('lang', Lang)
+    app.add_config_value('code-language-tabs-mode', 'html', True)

docs/shared/extensions/code_language_tabs/directives.py

@@ -0,0 +1,179 @@
+"""Adds directives for adding language tabs in the docs."""
+from docutils import nodes
+from sphinx.util.docutils import SphinxDirective
+from sphinx.util.nodes import set_source_info
+
+
+class CodeLanguageTabsError(Exception):
+    """Exception for handling code language tab errors."""
+
+    pass
+
+
+#: Name of the html extension.
+extHtml = 'html'
+#: Name of markdown extension
+extMarkdown = 'markdown'
+#: Toggle extension to off.
+extOff = 'off'
+#: Setting name.
+configSettingName = 'code-language-tabs-mode'
+#: Literal mode.
+modeLiteral = 'literal'
+#: Mixed mode.
+modeMixed = 'mixed'
+
+#: List of language names available.
+_lang_names = {
+    # ****************************************************************
+    # HTML has its own implementation of this due to technicalion reasons
+    # ****************************************************************
+    'sql': 'SQL',
+    'dotnet': '.NET',
+    'xml': 'XML',
+    'json': 'JSON',
+    'http': 'HTTP',
+    'md': 'Markdown',
+    'yaml': 'YAML',
+    'ini': 'ini',
+    'spacy': 'spaCy'
+}
+
+
+def _format_language_name(lang: str) -> str:
+    """Format a language name.
+
+    :param lang: Language name to format
+    :return: List of language names
+    """
+    return _lang_names[lang] if lang in _lang_names else lang.capitalize()
+
+
+def _get_page_language(node: nodes.Element, mode: str) -> str:
+    """Get current page language.
+
+    :param node: Current node
+    :param mode: Mode for string
+    :return: language item of node list
+    """
+    return node['language' if mode == modeLiteral else 'tabs-lang']
+
+
+def _format_markdown_node(node: nodes.Element, mode: str) -> nodes.Element:
+    """Format a markdown node.
+
+    :param node: Current node
+    :param mode: Current mode
+    :return: Resulting section of the document
+    """
+    lang = _get_page_language(node, mode)
+    name = _format_language_name(lang)
+    title = nodes.title(name, name)
+    section = nodes.section('', title, node)
+    section['ids'] = ['clt-section-id-' + lang]
+    return section
+
+
+class CodeLanguageTabs(SphinxDirective):
+    """Add code language tabs directive to Sphinx."""
+
+    #: (dict) contains the available options for this directive.
+    option_spec = {}
+    #: (bool) defines the existence of content for this instance.
+    has_content = True
+
+    def run(self):
+        """Apply directive changes when directive is found.
+
+        .. todo:: This is a little complex for my taste.
+        """
+        node = nodes.compound()
+        node.document = self.state.document
+        set_source_info(self, node)
+
+        self.state.nested_parse(self.content, self.content_offset,
+                                node)
+
+        if self.config[configSettingName] == extOff:
+            return node.children
+        else:
+            if not node.children:
+                raise CodeLanguageTabsError(
+                    'code-language-tabs directive may not be empty')
+
+            mode = self._get_directive_mode(node[0])
+            if len(node) == 1:
+                return (node.children
+                        if mode == modeLiteral else node[0].children)
+
+            for i in range(1, len(node)):
+                self._validate_against_mode(node[i], mode)
+
+            if self.config[configSettingName] == extMarkdown:
+                for child in node:
+                    new = _format_markdown_node(child, mode)
+                    node.replace(child, new)
+            else:
+                node['classes'] = ['js-code-language-tabs']
+                if mode == modeLiteral:
+                    node['classes'].append('js-code-language-tabs--literal')
+
+            return [node]
+
+    def _get_directive_mode(self, node: nodes.Element) -> str:
+        if self._is_literal_block(node):
+            return modeLiteral
+        elif self._is_mixed_content(node):
+            return modeMixed
+        else:
+            raise
+
+    def _validate_against_mode(self, node: nodes.Element, mode: str) -> None:
+        if mode == modeLiteral:
+            if not self._is_literal_block(node):
+                raise CodeLanguageTabsError(
+                    ('The code-language-tabs directive must'
+                     ' contain only literal_block children'))
+        else:
+            if not self._is_mixed_content(node):
+                raise CodeLanguageTabsError(
+                    ('The code-language-tabs directive must'
+                     ' contain only ".. lang:: <language>" children'))
+
+    def _is_literal_block(self, node: nodes.Element) -> bool:
+        return isinstance(node, nodes.literal_block)
+
+    def _is_mixed_content(self, node: nodes.Element) -> bool:
+        return isinstance(node, nodes.compound) and 'tabs-lang' in node
+
+
+class Lang(SphinxDirective):
+    """Add lang directive to Sphinx."""
+
+    #: Current option spec
+    option_spec = {}
+    #: Does the directive have content
+    has_content = True
+    #: Number of required arguments
+    required_arguments = 1
+
+    def run(self):
+        """Run build code when lang directive is found."""
+        lang = self.arguments[0]
+        node = nodes.compound()
+
+        node.document = self.state.document
+        set_source_info(self, node)
+        self.state.nested_parse(self.content, self.content_offset,
+                                node)
+
+        if self.config[configSettingName] == extOff:
+            return node.children
+        else:
+            if not node.children:
+                raise CodeLanguageTabsError('lang directive may not be empty')
+
+            node['classes'] = ['language-' + lang]
+            node['tabs-lang'] = lang
+
+            return [node]