diff --git a/pyclient/docs/.gitignore b/pyclient/docs/.gitignore new file mode 100644 index 00000000000..a52866727e4 --- /dev/null +++ b/pyclient/docs/.gitignore @@ -0,0 +1 @@ +source/code \ No newline at end of file diff --git a/pyclient/docs/source/conf.py b/pyclient/docs/source/conf.py index ce787c5aa08..720abfcb158 100644 --- a/pyclient/docs/source/conf.py +++ b/pyclient/docs/source/conf.py @@ -10,10 +10,9 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # -import os -import sys - -sys.path.insert(0, os.path.abspath('..')) +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) # -- Project information ----------------------------------------------------- @@ -29,8 +28,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.napoleon', 'sphinx.ext.todo', 'sphinx.ext.viewcode', 'sphinx.ext.autodoc', - "sphinx_autodoc_typehints"] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.todo', 'sphinx.ext.viewcode', "sphinx_autodoc_typehints"] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -38,7 +36,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["proto"] +exclude_patterns = [] # -- Options for HTML output ------------------------------------------------- @@ -52,10 +50,29 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +# Custom CSS files html_css_files = ['custom.css'] + +# Theme options +# see https://alabaster.readthedocs.io/en/latest/customization.html +# see https://github.com/bitprophet/alabaster/blob/master/alabaster/theme.conf html_theme_options = { + #'logo' : 'deephaven.png', + #'logo_name' : 'Deephaven', 'page_width': '80%', 'sidebar_width': '35%', } +# A boolean that decides whether module names are prepended to all object names (for object types where a “module” of some kind is defined), e.g. for py:function directives. Default is True. add_module_names = False + +######################################################################################################################################################################### + +import pydeephaven +docs_title = "Deephaven Python Client API modules." +package_roots = [pydeephaven] +package_excludes = ['._', 'proto'] + +import dh_sphinx +dh_sphinx.gen_sphinx_modules(docs_title, package_roots, package_excludes) + diff --git a/pyclient/docs/source/genindex.rst b/pyclient/docs/source/genindex.rst new file mode 100644 index 00000000000..55e4997997e --- /dev/null +++ b/pyclient/docs/source/genindex.rst @@ -0,0 +1,2 @@ +Index +##### diff --git a/pyclient/docs/source/index.rst b/pyclient/docs/source/index.rst index 65da50aefc5..61ecc6f86ca 100644 --- a/pyclient/docs/source/index.rst +++ b/pyclient/docs/source/index.rst @@ -1,7 +1,11 @@ -.. Deephaven Python Client API documentation master file, created by - sphinx-quickstart on Thu Aug 19 12:27:56 2021. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. + +.. toctree:: + :name: mastertoc + :hidden: + + modules.rst + py-modindex.rst + genindex.rst Deephaven Python Client API Documentation @@ -9,7 +13,7 @@ Deephaven Python Client API Documentation Deephaven Python Client (pydeephaven) is a Python API built on top of Deephaven’s highly efficient OpenAPI which is based on gRPC and Apache Arrow. It allows Python applications to remotely connect to Deephaven data servers, export/import data with the server, run Python scripts on the server, and execute powerful queries on data tables. -Because Deephaven data servers and Deephaven clients including pydeephaven exchange data in the Apache Arrow format, pydeephaven is able to leverage ‘pyarrow’ - the Python bindings of Arrow (ttps://arrow.apache.org/docs/python/) for data representation and integration with other data analytic tools such as NumPy, Pandas, etc. +Because Deephaven data servers and Deephaven clients including pydeephaven exchange data in the Apache Arrow format, pydeephaven is able to leverage ‘pyarrow’ - the Python bindings of Arrow (https://arrow.apache.org/docs/python/) for data representation and integration with other data analytic tools such as NumPy, Pandas, etc. Examples: >>> from pydeephaven import Session @@ -21,17 +25,3 @@ Examples: >>> df = joined_table.snapshot().to_pandas() >>> print(df) >>> session.close() - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - modules - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/pyclient/docs/source/modules.rst b/pyclient/docs/source/modules.rst index 5260fd45e01..cb5a5490a8d 100644 --- a/pyclient/docs/source/modules.rst +++ b/pyclient/docs/source/modules.rst @@ -1,4 +1,11 @@ + +Python Modules +############## + +Deephaven python modules. + .. toctree:: - :maxdepth: 4 + :glob: + - pydeephaven + ./code/pydeephaven \ No newline at end of file diff --git a/pyclient/docs/source/py-modindex.rst b/pyclient/docs/source/py-modindex.rst new file mode 100644 index 00000000000..c52a8313f68 --- /dev/null +++ b/pyclient/docs/source/py-modindex.rst @@ -0,0 +1,3 @@ +Module Index +############ + diff --git a/pyclient/docs/source/pydeephaven.rst b/pyclient/docs/source/pydeephaven.rst deleted file mode 100644 index ba135bbc1bf..00000000000 --- a/pyclient/docs/source/pydeephaven.rst +++ /dev/null @@ -1,55 +0,0 @@ -pydeephaven package -=================== - -.. toctree:: - :maxdepth: 2 - -pydeephaven.constants module ----------------------------- - -.. automodule:: pydeephaven.constants - :members: - :show-inheritance: - -pydeephaven.dherror module --------------------------- - -.. automodule:: pydeephaven.dherror - :members: - :show-inheritance: - -pydeephaven.query module ------------------------- - -.. automodule:: pydeephaven.query - :members: - :show-inheritance: - -pydeephaven.session module --------------------------- - -.. automodule:: pydeephaven.session - :members: - :show-inheritance: - :special-members: __init__ - -pydeephaven.table module ------------------------- - -.. automodule:: pydeephaven.table - :members: - :show-inheritance: - -pydeephaven._table_interface module ------------------------------------ - -.. automodule:: pydeephaven._table_interface - :members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: pydeephaven - :members: - :show-inheritance: diff --git a/sphinx/lib/dh_sphinx.py b/sphinx/lib/dh_sphinx.py new file mode 100644 index 00000000000..c16a60d0d21 --- /dev/null +++ b/sphinx/lib/dh_sphinx.py @@ -0,0 +1,96 @@ + +import os +import shutil +import pkgutil + +def glob_package_names(packages): + rst = [] + + for package in packages: + rst.append(package.__name__) + + if hasattr(package,"__path__"): + for importer, modname, ispkg in pkgutil.walk_packages(path=package.__path__, prefix=package.__name__+'.', onerror=lambda x: None): + rst.append(modname) + + return rst + + +def _add_package(tree, package): + n = package[0] + + if n not in tree: + tree[n] = {} + + if len(package) > 1: + _add_package(tree[n],package[1:]) + + +def package_tree(package_names): + rst = {} + for pn in package_names: + spn = pn.split('.') + _add_package(rst, spn) + return rst + + +def make_rst_tree(package, tree): + package_name = ".".join(package) + + if len(tree) == 0: + toctree = "" + else: + toctree = ".. toctree::\n" + for k in tree: + p = package.copy() + p.append(k) + pn = ".".join(p) + toctree += "%s%s <%s>\n"%(" "*4,k,pn) + + rst = "%s\n%s\n\n%s\n.. automodule:: %s\n :members:\n :show-inheritance:\n :undoc-members:\n\n"%(package_name,"="*len(package_name),toctree,package_name) + + if len(package) > 0: + filename = f"code/{package_name}.rst" + + with open(filename,"w") as file: + file.write(rst) + + for k,v in tree.items(): + p = package.copy() + p.append(k) + make_rst_tree(p, v) + +def make_rst_modules(docs_title, package_roots): + rst = f''' +Python Modules +############## + +{docs_title} + +.. toctree:: + :glob: + +''' + + for pr in package_roots: + rst += "\n%s./code/%s"%(" "*4,pr.__name__) + + filename = "modules.rst" + + with open(filename,"w") as file: + file.write(rst) + +def gen_sphinx_modules(docs_title, package_roots, package_excludes): + pn = glob_package_names(package_roots) + pn = [p for p in pn if not any(exclude in p for exclude in package_excludes)] + pt = package_tree(pn) + + if os.path.exists("code"): + shutil.rmtree("code") + os.mkdir("code") + + make_rst_modules(docs_title, package_roots) + make_rst_tree([],pt) + + + diff --git a/sphinx/source/conf.py b/sphinx/source/conf.py index 1bc2ce92220..269665ac189 100644 --- a/sphinx/source/conf.py +++ b/sphinx/source/conf.py @@ -19,12 +19,16 @@ project = 'Deephaven' copyright = '2021, Deephaven Data Labs' author = 'Deephaven Data Labs' + +# The full version, including alpha/beta/rc tags +#release = '0.0.1' + # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.napoleon'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.todo', 'sphinx.ext.viewcode', "sphinx_autodoc_typehints"] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -64,106 +68,12 @@ ######################################################################################################################################################################### -import os -import shutil -import pkgutil - -def glob_package_names(packages): - rst = [] - - for package in packages: - rst.append(package.__name__) - - if hasattr(package,"__path__"): - for importer, modname, ispkg in pkgutil.walk_packages(path=package.__path__, prefix=package.__name__+'.', onerror=lambda x: None): - rst.append(modname) - - return rst - - -def _add_package(tree, package): - n = package[0] - - if n not in tree: - tree[n] = {} - - if len(package) > 1: - _add_package(tree[n],package[1:]) - - -def package_tree(package_names): - rst = {} - for pn in package_names: - spn = pn.split('.') - _add_package(rst, spn) - return rst - - -def make_rst_tree(package, tree): - package_name = ".".join(package) - - if len(tree) == 0: - toctree = "" - else: - toctree = ".. toctree::\n" - for k in tree: - p = package.copy() - p.append(k) - pn = ".".join(p) - toctree += "%s%s <%s>\n"%(" "*4,k,pn) - - rst = "%s\n%s\n\n%s\n.. automodule:: %s\n :members:\n :undoc-members:\n\n"%(package_name,"="*len(package_name),toctree,package_name) - - if len(package) > 0: - filename = f"code/{package_name}.rst" - - with open(filename,"w") as file: - file.write(rst) - - for k,v in tree.items(): - p = package.copy() - p.append(k) - make_rst_tree(p, v) - - -_rst_modules = ''' -Python Modules -############## - -Deephaven python modules. - -.. toctree:: - :glob: - -''' - -def make_rst_modules(package_roots): - rst = _rst_modules - - for pr in package_roots: - rst += "\n%s./code/%s"%(" "*4,pr.__name__) - - filename = "modules.rst" - - with open(filename,"w") as file: - file.write(rst) - - - import deephaven import jpy +docs_title = "Deephaven python modules." package_roots = [jpy, deephaven] -pn = glob_package_names(package_roots) -# remove private modules -pn = [p for p in pn if not '._' in p] -pt = package_tree(pn) - -if os.path.exists("code"): - shutil.rmtree("code") -os.mkdir("code") - -make_rst_modules(package_roots) -make_rst_tree([],pt) - +package_excludes = ['._'] +import dh_sphinx +dh_sphinx.gen_sphinx_modules(docs_title, package_roots, package_excludes) diff --git a/sphinx/sphinx.gradle b/sphinx/sphinx.gradle index 523e9c464fd..6b3d9c2e976 100644 --- a/sphinx/sphinx.gradle +++ b/sphinx/sphinx.gradle @@ -7,17 +7,34 @@ plugins { description = 'Generates docs for the python libraries provided in Deephaven Core' evaluationDependsOn ':Integrations' +evaluationDependsOn ':pyclient' + +def copyPyClientWhl = tasks.register('copyPyClientWhl', Sync) { + from project(':pyclient').tasks.named('buildPyClient').get().outputs.files + into layout.buildDirectory.file('sphinx-image/wheel/') +} + +def copySphinxLib = tasks.register('copySphinxLib', Sync) { + from "$rootDir/sphinx/lib/" + into layout.buildDirectory.file('sphinx-image/lib') +} def sphinxDockerfile = tasks.register('sphinxDockerfile', Dockerfile) { destFile.set layout.buildDirectory.file('sphinx-image/Dockerfile') from 'deephaven/runtime-base:local-build' + copyFile "./wheel", "/wheel" + copyFile "./lib", "/usr/lib/python3/dist-packages/" + runCommand '''set -eux; \\ - pip3 install sphinx==3.5.4 sphinx-autodoc-typehints==1.12.0 pyarrow==5.0.0 protobuf==3.17.3 grpcio==1.39.0 bitstring==3.1.9 + pip3 install sphinx==3.5.4 sphinx-autodoc-typehints==1.12.0 pyarrow==5.0.0 protobuf==3.17.3 grpcio==1.39.0 bitstring==3.1.9 /wheel/*.whl ''' } + def sphinxImage = Docker.registerDockerImage(project, 'sphinx') { inputs.files sphinxDockerfile.get().outputs.files + inputs.files copyPyClientWhl.get().outputs.files + inputs.files copySphinxLib.get().outputs.files inputDir.set layout.buildDirectory.dir('sphinx-image') inputs.files project(':Integrations').tasks.findByName('buildDeephavenPython').outputs.files // deephaven/runtime-base images.add('deephaven/sphinx:local-build')