From 5d118fb47ada6425738a79e75467e6b61d2d8c58 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 15:28:40 +0100 Subject: [PATCH 01/16] Try adding docs rtd check on kedro datasets Signed-off-by: Merel Theisen --- .github/workflows/check-plugin.yml | 4 + .github/workflows/run-docs-build.yml | 25 + Makefile | 3 + kedro-datasets/docs/build-docs.sh | 14 + .../source/_static/css/qb1-sphinx-rtd.css | 493 ++++++++++++++++ .../source/_static/css/theme-overrides.css | 35 ++ .../source/_templates/autosummary/base.rst | 5 + .../source/_templates/autosummary/class.rst | 32 + .../source/_templates/autosummary/module.rst | 56 ++ .../docs/source/_templates/breadcrumbs.html | 94 +++ .../docs/source/_templates/layout.html | 8 + kedro-datasets/docs/source/conf.py | 558 ++++++++++++++++++ kedro-datasets/docs/source/index.rst | 22 + kedro-datasets/docs/source/kedro_datasets.rst | 56 ++ .../docs/source/puppeteer-config.json | 4 + kedro-datasets/docs/source/robots.txt | 5 + 16 files changed, 1414 insertions(+) create mode 100644 .github/workflows/run-docs-build.yml create mode 100755 kedro-datasets/docs/build-docs.sh create mode 100644 kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css create mode 100644 kedro-datasets/docs/source/_static/css/theme-overrides.css create mode 100644 kedro-datasets/docs/source/_templates/autosummary/base.rst create mode 100644 kedro-datasets/docs/source/_templates/autosummary/class.rst create mode 100644 kedro-datasets/docs/source/_templates/autosummary/module.rst create mode 100644 kedro-datasets/docs/source/_templates/breadcrumbs.html create mode 100644 kedro-datasets/docs/source/_templates/layout.html create mode 100644 kedro-datasets/docs/source/conf.py create mode 100644 kedro-datasets/docs/source/index.rst create mode 100644 kedro-datasets/docs/source/kedro_datasets.rst create mode 100644 kedro-datasets/docs/source/puppeteer-config.json create mode 100644 kedro-datasets/docs/source/robots.txt diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml index 045c36c13..5b7fcbe87 100644 --- a/.github/workflows/check-plugin.yml +++ b/.github/workflows/check-plugin.yml @@ -61,6 +61,10 @@ jobs: if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version != '3.10' run: | make test-no-spark + - name: RTD build for kedro-datasets + if: inputs.plugin == 'kedro-datasets' && matrix.python-version == '3.10' + run: | + make rtd lint: defaults: diff --git a/.github/workflows/run-docs-build.yml b/.github/workflows/run-docs-build.yml new file mode 100644 index 000000000..79ee833f4 --- /dev/null +++ b/.github/workflows/run-docs-build.yml @@ -0,0 +1,25 @@ +name: Run Doc builds on Kedro-Datasets + +on: + push: + branches: + - main + - try-rtd-build + paths-ignore: + - "kedro-airflow/**" + - "kedro-docker/**" + - "kedro-telemetry/**" + pull_request: + branches: + - main + - try-rtd-build + paths-ignore: + - "kedro-airflow/**" + - "kedro-docker/**" + - "kedro-telemetry/**" + +jobs: + datasets-test: + uses: ./.github/workflows/check-plugin.yml + with: + plugin: kedro-datasets diff --git a/Makefile b/Makefile index 074095ba7..1c6c7e478 100644 --- a/Makefile +++ b/Makefile @@ -60,3 +60,6 @@ test-no-spark-sequential: # kedro-datasets/snowflake tests skipped from default scope test-snowflake-only: cd kedro-datasets && pytest tests --no-cov --numprocesses 1 --dist loadfile -m snowflake + +rtd: + cd kedro-datasets && python -m sphinx -WETan -j auto -D language=en -b linkcheck -d _build/doctrees docs/source _build/linkcheck diff --git a/kedro-datasets/docs/build-docs.sh b/kedro-datasets/docs/build-docs.sh new file mode 100755 index 000000000..d55076e11 --- /dev/null +++ b/kedro-datasets/docs/build-docs.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e + +# Exit script if you try to use an uninitialized variable. +set -o nounset + +action=$1 + +if [ "$action" == "linkcheck" ]; then + sphinx-build -WETan -j auto -D language=en -b linkcheck -d docs/build/doctrees docs/source docs/build/linkcheck +elif [ "$action" == "docs" ]; then + sphinx-build -WETa -j auto -D language=en -b html -d docs/build/doctrees docs/source docs/build/html +fi diff --git a/kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css b/kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css new file mode 100644 index 000000000..3f11d0cee --- /dev/null +++ b/kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css @@ -0,0 +1,493 @@ +@import url("https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600&display=swap"); + +html, body.wy-body-for-nav { + margin: 0; + padding: 0; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + font-family: 'Inter', sans-serif !important; + font-weight: 400; + line-height: 2rem; +} + +html { + font-size: 62.5%; +} + +:focus { + outline-color: #006dff; +} + +body.wy-body-for-nav { + font-size: 1.4289rem; + background: rgb(250, 250, 250) !important; + color: #1e1e21; + position: relative; +} + +.wy-side-nav-search { + text-align: left; +} + +.wy-side-nav-search input[type=text] { + display: block; + box-sizing: border-box; + width: 100%; + padding: 6px 12px; + color: #666; + background-color: #fff; + font-family: 'Inter', sans-serif !important; + font-size: 1.4289rem; + border: 1px #ccc solid; + border-radius: 2px; + transition: all ease 0.15s; + box-shadow: none; +} + +.wy-side-nav-search input[type=text]:focus { + border-color: #888; + color: #333; +} + +.wy-body-for-nav .wy-nav-side { + position: fixed; + top: 0; + bottom: 0; + left: 0; + padding-bottom: 0; + width: 300px; + overflow-x: hidden; + min-height: 100%; + background: white; + z-index: 200; + box-shadow: 0 2px 5px 0 rgba(0, 0, 0, .05); +} + +.wy-body-for-nav .wy-side-scroll { + display: flex; + flex-direction: column; + width: 100%; + position: relative; + overflow-x: initial; + overflow-y: initial; +} + +.wy-body-for-nav .wy-side-nav-search { + width: 100%; + background: none; + margin: 0; + padding: 0 20px; +} +.wy-body-for-nav .wy-side-nav-search a { + display: inline-flex; + flex-direction: row-reverse; + align-items: center; + font-size: 4rem; + margin: 0; + padding: 0; + height: 7rem; + line-height: 7rem; + color: black; +} +.wy-body-for-nav .wy-side-nav-search a:before { + display: none; +} +.wy-body-for-nav .wy-side-nav-search a img.logo { + width: 4.5rem; + height: auto; + margin: 0 0.3rem 0 -0.6rem; + padding: 0; +} +.wy-body-for-nav .wy-side-nav-search>div.version { + color: #555; + display: inline-block; + margin-left: 0.5em; + font-size: 1.4rem; +} + +.wy-body-for-nav .wy-menu-vertical { + width: 300px; + margin: 0; + padding: 0px 20px 64px 20px; + overflow-y: auto; + height: 100%; +} + +.wy-body-for-nav .wy-menu-vertical p.caption { + margin: 1.5em 0 0.2em; + padding: 0; + font-size: 1.4289rem; + color: #161616; + font-weight: normal; + text-transform: none; +} + +.wy-body-for-nav .wy-menu-vertical p.caption:first-child { + margin-top: 1em; +} + +.wy-body-for-nav .wy-menu-vertical li.on a, .wy-body-for-nav .wy-menu-vertical li.current>a { + border-top: none; + border-bottom: none; +} + +.wy-body-for-nav .wy-menu-vertical li.current { + background: none; +} + +.wy-body-for-nav .wy-menu-vertical li { + margin: 0; + padding: 0 0 0 20px; +} + +.wy-body-for-nav .wy-menu-vertical li a { + display: block; + margin: 0; + padding: 0.9rem 0 !important; + font-size: 1.4289rem; + line-height: 1.2; + color: #222; + background: none !important; +} + +.wy-menu-vertical li.toctree-l1>a { + font-size: 1.4289rem; +} + +.rst-content.style-external-links a.reference.external:after { + color: inherit; + opacity: 0.8; +} + +.rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal { + color: #e72000; +} + +.highlight .nv { + color: #bd269e; +} + +.highlight .c1, +.highlight .sd { + color: #147087; +} + +.highlight .nn { + color: #0e84b5; +} + +.highlight .nn { + color: #0047bf; +} + +.wy-body-for-nav .wy-menu-vertical a:hover { + color: #000; +} + +.wy-body-for-nav li span.toctree-expand { + margin-left: -20px; +} + +.wy-body-for-nav .toctree-expand:before { + font-size: 15px; + margin-right: 5px; +} + +.wy-body-for-nav .wy-nav-content-wrap { + margin-left: 300px; + background: #fafafa; +} + +.wy-body-for-nav .wy-nav-content { + padding: 0; + max-width: initial; +} + +.wy-body-for-nav .wy-breadcrumbs { + display: -webkit-box; + display: -ms-flexbox; + display: flex; + -webkit-box-align: center; + -ms-flex-align: center; + align-items: center; + width: 100%; + height: 7rem; + padding-left: 3.2rem; + background: white; + background-image: initial; + background-position-x: initial; + background-position-y: initial; + background-size: initial; + background-repeat-x: initial; + background-repeat-y: initial; + background-attachment: initial; + background-origin: initial; + background-clip: initial; + background-color: white; + box-shadow: 0 2px 5px 0 rgba(0, 0, 0, .05); +} + +.wy-body-for-nav .wy-breadcrumbs + hr { + display: none; +} +.wy-body-for-nav .wy-breadcrumbs li { + font-size: 1.8rem; +} + +.wy-body-for-nav .wy-breadcrumbs li:not(:first-child) { + margin-left: 6px; +} + +.wy-body-for-nav .wy-breadcrumbs li.wy-breadcrumbs-aside { + margin-left: 24px; +} + +.wy-body-for-nav .wy-nav-content .document { + padding: 10.2rem 3.2rem 2rem; + max-width: 100rem; +} + +.kedro-logo { + margin-bottom: 3rem; +} + +.wy-body-for-nav footer { + padding: 32px; +} + +.wy-body-for-nav .wy-body-for-nav { + background: #fafafa !important; + background-image: none; + background-size: initial; +} + +.wy-body-for-nav .wy-menu-vertical li.current a:hover { + background: none; + font-weight: 600; +} + +.wy-menu-vertical a span.toctree-expand { + color: #555 !important; +} + +.wy-menu-vertical a:hover span.toctree-expand { + color: #000 !important; +} + +.wy-body-for-nav .wy-menu-vertical li.current a { + border-right: none; + font-weight: normal; +} + +.wy-body-for-nav .wy-menu-vertical li.current a.current { + font-weight: bold; +} + +.wy-body-for-nav .toctree-l2, +.wy-body-for-nav .toctree-l3, +.wy-body-for-nav .toctree-l4, +.wy-body-for-nav .toctree-l5 { + padding-left: 8px !important; +} + +.wy-body-for-nav .toctree-l4 a { + word-break: break-word; +} + +.wy-plain-list-disc li, +.rst-content .section ul li, +.rst-content .toctree-wrapper ul li, +article ul li { + list-style: square; + margin-top: 0.35em; + margin-bottom: 0.35em; +} + +.rst-content .toctree-wrapper ul li { + margin-top: 0.5em; + margin-bottom: 0.5em; +} + +h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend { + font-family: 'Inter', sans-serif !important; + margin: 1.2em 0 1em 0; + line-height: 1.2em; + color: #272c2e; + font-weight: normal; +} + +.wy-body-for-nav .document > div > .section > *:not(:empty):first-of-type { + margin-top: 0; +} + +.wy-body-for-nav h1 { + font-size: 2.6rem; + letter-spacing: -0.3px; +} + +.wy-body-for-nav h2 { + font-size: 2.3rem; +} + +.wy-body-for-nav h3 { + font-size: 2.1rem; +} + +.wy-body-for-nav h4 { + font-size: 1.95rem; +} + +.wy-body-for-nav h5 { + font-size: 1.625rem; +} + +.wy-body-for-nav h6 { + font-size: 1.4625rem; +} + +.wy-body-for-nav p { + font-size: 100%; + margin: 0 0 1em 0; + line-height: 1.7; +} + +.wy-body-for-nav p + details { + margin: -4px 0 15px; +} + +.wy-body-for-nav blockquote { + margin: 1em 0; + padding-left: 1em; + border-left: 4px solid #ddd; + color: #6a6a6a; +} + +footer span.commit code, +footer span.commit .rst-content tt, +.rst-content footer span.commit tt { + color: #6f7070; +} + +.rst-content tt, .rst-content tt, .rst-content code { + padding: 0.16em 0.48em; +} + +.wy-body-for-nav .rst-content a, .wy-body-for-nav footer a { + font-family: 'Inter', sans-serif !important; + font-size: inherit; + color: #006ea7; + text-decoration: none; +} + +.wy-body-for-nav .rst-content a:visited, .wy-body-for-nav footer a:visited { + color: #447087; + text-decoration-color: #739eb4; +} + +.wy-body-for-nav .rst-content .btn, .wy-body-for-nav footer .btn, +.wy-body-for-nav .rst-content .btn:hover, .wy-body-for-nav footer .btn:hover, +.wy-body-for-nav .rst-content .headerlink, .wy-body-for-nav .rst-content .headerlink:hover, +.wy-body-for-nav .wy-breadcrumbs li a, +.wy-nav-content .wy-breadcrumbs .fa-github, .wy-nav-content .wy-breadcrumbs .fa-github:hover { + text-decoration: none; +} + +.wy-body-for-nav .rst-content a, .wy-body-for-nav footer a, +.wy-body-for-nav .rst-content a:hover, .wy-body-for-nav footer a:hover { + text-decoration: underline; + text-underline-position: under; + text-decoration-color: #b7d9f2; +} + +.rst-content a code span { + text-decoration: underline; + text-decoration-color: #fff; +} + +.rst-content a code { + border-bottom: 1px solid #ff8c8c; +} + +.wy-body-for-nav .wy-nav-top { + padding: 5px 20px; + background: white; + color: black; + box-shadow: 0 2px 5px 0 rgba(0, 0, 0, 0.05); +} + +.wy-body-for-nav .wy-nav-top i { + transform: translateY(6px); +} + +.wy-body-for-nav .wy-nav-top a { + font-size: 2.8rem; + color: black !important; +} + +.wy-side-scroll p.caption[role="heading"] span.caption-text { + font-weight: 700; +} + +.wy-nav-content h1, +.wy-nav-content p.caption[role="heading"] .caption-text { + font-size: 1.7857rem; + font-weight: 700; +} + + +@media screen and (max-width: 768px) { + .wy-body-for-nav .wy-nav-side { + transform: translate(-300px, 0); + transition: all ease 0.3s; + } + .wy-body-for-nav .wy-nav-side.shift { + width: 85%; + transform: translate(0, 0); + } + .wy-body-for-nav .wy-nav-content-wrap { + margin-left: 0; + transform: translate(0, 0); + transition: all ease 0.3s; + } + .wy-body-for-nav .wy-nav-content-wrap.shift { + position: relative; + left: 0; + transform: translate(85%, 0); + } + .wy-body-for-nav .wy-breadcrumbs { + display: none; + } + .wy-body-for-nav .wy-nav-content .document { + padding: 20px; + } +} + +@media screen and (min-width: 1600px) { + .wy-body-for-nav .wy-nav-side { + width: 350px; + } + .wy-body-for-nav .wy-nav-content-wrap { + margin-left: 350px; + } + html { + font-size: 70%; + } +} + +/* Fix Read The Docs side-effects */ +.wy-body-for-nav .rst-versions { + font-size: 16px; + line-height: 1; +} + +.wy-breadcrumbs { + position: fixed; + z-index: 2; +} + +/* Offset target anchor to below the fixed header */ +:target:before { + display: block; + content: ' '; + width: 1px; +} diff --git a/kedro-datasets/docs/source/_static/css/theme-overrides.css b/kedro-datasets/docs/source/_static/css/theme-overrides.css new file mode 100644 index 000000000..bdf28d927 --- /dev/null +++ b/kedro-datasets/docs/source/_static/css/theme-overrides.css @@ -0,0 +1,35 @@ +/* override table width restrictions */ +@media screen and (min-width: 767px) { + .wy-table-responsive table td { + white-space: normal; + } + + .wy-table-responsive { + overflow: visible; + } +} + +/* override the table font-size and line-height */ +html.writer-html5 .rst-content .wy-table-responsive table.docutils th > p, +html.writer-html5 .rst-content .wy-table-responsive table.docutils td > p { + font-size: 1em; + line-height: 1em; +} + +img[alt^="mermaid-"] { + max-width: 600px; +} + +.rst-content .important { + background: #ffedcc; +} +.rst-content .important .admonition-title { + background-color: #f0b37e; +} + +/* Ensure the section title is visible when linked via a hash in the URL */ +:target:before { + content: ""; + display: block; + height: 80px; +} diff --git a/kedro-datasets/docs/source/_templates/autosummary/base.rst b/kedro-datasets/docs/source/_templates/autosummary/base.rst new file mode 100644 index 000000000..b7556ebf7 --- /dev/null +++ b/kedro-datasets/docs/source/_templates/autosummary/base.rst @@ -0,0 +1,5 @@ +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. auto{{ objtype }}:: {{ objname }} diff --git a/kedro-datasets/docs/source/_templates/autosummary/class.rst b/kedro-datasets/docs/source/_templates/autosummary/class.rst new file mode 100644 index 000000000..10c8ff8be --- /dev/null +++ b/kedro-datasets/docs/source/_templates/autosummary/class.rst @@ -0,0 +1,32 @@ +{{ fullname | escape | underline }} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :undoc-members: + :inherited-members: + + {% block attributes %} + {% if attributes %} + .. rubric:: Attributes + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block methods %} + {% if methods %} + .. rubric:: Methods + + .. autosummary:: + {% for item in all_methods %} + {%- if not item.startswith('_') %} + ~{{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/kedro-datasets/docs/source/_templates/autosummary/module.rst b/kedro-datasets/docs/source/_templates/autosummary/module.rst new file mode 100644 index 000000000..a496ca3f5 --- /dev/null +++ b/kedro-datasets/docs/source/_templates/autosummary/module.rst @@ -0,0 +1,56 @@ +{{ fullname | escape | underline }} + +.. rubric:: Description + +.. automodule:: {{ fullname }} + + {% block functions %} + {% if functions %} + .. rubric:: Functions + + .. autosummary:: + :toctree: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: Classes + + .. autosummary:: + :toctree: + :template: autosummary/class.rst + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: Exceptions + + .. autosummary:: + :toctree: + :template: autosummary/class.rst + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/kedro-datasets/docs/source/_templates/breadcrumbs.html b/kedro-datasets/docs/source/_templates/breadcrumbs.html new file mode 100644 index 000000000..49fa4779f --- /dev/null +++ b/kedro-datasets/docs/source/_templates/breadcrumbs.html @@ -0,0 +1,94 @@ +{# Support for Sphinx 1.3+ page_source_suffix, but don't break old builds. #} + +{% if page_source_suffix %} +{% set suffix = page_source_suffix %} +{% else %} +{% set suffix = source_suffix %} +{% endif %} + +{# modification to enable custom github_url #} + +{% if meta is not defined or meta is none %} + {% set meta = {} %} +{% endif %} + +{% if github_url is defined %} + {% set _dummy = meta.update({'github_url': github_url}) %} +{% endif %} + +{# // modification to enable custom github_url #} + +{% if meta is defined and meta is not none %} +{% set check_meta = True %} +{% else %} +{% set check_meta = False %} +{% endif %} + +{% if check_meta and 'github_url' in meta %} +{% set display_github = True %} +{% endif %} + +{% if check_meta and 'bitbucket_url' in meta %} +{% set display_bitbucket = True %} +{% endif %} + +{% if check_meta and 'gitlab_url' in meta %} +{% set display_gitlab = True %} +{% endif %} + +
+ + + + {% if (theme_prev_next_buttons_location == 'top' or theme_prev_next_buttons_location == 'both') and (next or prev) %} + + {% endif %} +
+
diff --git a/kedro-datasets/docs/source/_templates/layout.html b/kedro-datasets/docs/source/_templates/layout.html new file mode 100644 index 000000000..ecdde06f1 --- /dev/null +++ b/kedro-datasets/docs/source/_templates/layout.html @@ -0,0 +1,8 @@ +{% extends "!layout.html" %} + +{%- block extrahead %} + +{% endblock %} diff --git a/kedro-datasets/docs/source/conf.py b/kedro-datasets/docs/source/conf.py new file mode 100644 index 000000000..ca28f16c5 --- /dev/null +++ b/kedro-datasets/docs/source/conf.py @@ -0,0 +1,558 @@ +#!/usr/bin/env python3 +# +# Kedro documentation build configuration file, created by +# sphinx-quickstart on Mon Dec 18 11:31:24 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. +from __future__ import annotations + +import importlib +import os +import re +import sys +from inspect import getmembers, isclass, isfunction +from pathlib import Path + +from click import secho, style + +from kedro import __version__ as release + +# -- Project information ----------------------------------------------------- + +project = "kedro" +author = "kedro" + +# The short X.Y version. +version = re.match(r"^([0-9]+\.[0-9]+).*", release).group(1) + + +# -- General configuration --------------------------------------------------- +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx_autodoc_typehints", + "sphinx.ext.doctest", + "sphinx.ext.ifconfig", + "sphinx.ext.viewcode", + "sphinx_copybutton", + "sphinxcontrib.mermaid", + "myst_parser", + "notfound.extension", +] + +# enable autosummary plugin (table of contents for modules/classes/class +# methods) +autosummary_generate = True +autosummary_generate_overwrite = False +napoleon_include_init_with_doc = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] +html_static_path = ["_static"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = {".rst": "restructuredtext", ".md": "markdown"} + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [ + "**.ipynb_checkpoints", + "_templates", + "modules.rst", + "source", + "kedro_docs_style_guide.md", +] + + +type_targets = { + "py:class": ( + "object", + "bool", + "int", + "float", + "str", + "tuple", + "Any", + "Dict", + "dict", + "list", + "set", + "typing.Dict", + "typing.Iterable", + "typing.List", + "typing.Tuple", + "typing.Type", + "typing.Set", + "kedro.config.config.ConfigLoader", + "kedro.io.core.AbstractDataSet", + "kedro.io.AbstractDataSet", + "kedro.io.core.AbstractVersionedDataSet", + "kedro.io.core.DataSetError", + "kedro.io.core.Version", + "kedro.io.data_catalog.DataCatalog", + "kedro.io.memory_dataset.MemoryDataSet", + "kedro.io.partitioned_dataset.PartitionedDataSet", + "kedro.pipeline.pipeline.Pipeline", + "kedro.runner.runner.AbstractRunner", + "kedro.runner.parallel_runner._SharedMemoryDataSet", + "kedro.runner.parallel_runner._SharedMemoryDataset", + "kedro.framework.context.context.KedroContext", + "kedro.framework.startup.ProjectMetadata", + "abc.ABC", + "Path", + "pathlib.Path", + "PurePosixPath", + "pathlib.PurePosixPath", + "requests.auth.AuthBase", + "google.oauth2.credentials.Credentials", + "Exception", + "CONF_SOURCE", + "integer -- return number of occurrences of value", + "integer -- return first index of value.", + "kedro.extras.datasets.pandas.json_dataset.JSONDataSet", + "kedro_datasets.pandas.json_dataset.JSONDataSet", + "pluggy._manager.PluginManager", + "PluginManager", + "_DI", + "_DO", + "deltalake.table.Metadata", + # The statements below were added after subclassing UserDict in AbstractConfigLoader. + "None. Remove all items from D.", + "a shallow copy of D", + "a set-like object providing a view on D's items", + "a set-like object providing a view on D's keys", + "v, remove specified key and return the corresponding value.", + "None. Update D from dict/iterable E and F.", + "an object providing a view on D's values", + "(k, v), remove and return some (key, value) pair", + "D.get(k,d), also set D[k]=d if k not in D", + "None. Update D from mapping/iterable E and F.", + "DataCatalog" + ), + "py:data": ( + "typing.Any", + "typing.Callable", + "typing.Union", + "typing.Optional", + "typing.Tuple", + ), + "py:exc": ( + "ValueError", + "BadConfigException", + "MissingConfigException", + "DataSetError", + "ImportError", + "KedroCliError", + "Exception", + "TypeError", + "SyntaxError", + "CircularDependencyError", + "OutputNotUniqueError", + "ConfirmNotUniqueError", + "ParserError", + "DatasetError", + ), +} +# https://stackoverflow.com/questions/61770698/sphinx-nit-picky-mode-but-only-for-links-i-explicitly-wrote +nitpick_ignore = [(key, value) for key in type_targets for value in type_targets[key]] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" +here = Path(__file__).parent.absolute() + +# Theme options are theme-specific and customise the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +html_theme_options = {"collapse_navigation": False, "style_external_links": True} + +# html_extra_path used to define a path to robots.txt which is used by webcrawlers +# to ignore or allow certain links. +html_extra_path = [str(here / "robots.txt")] + +# Removes, from all docs, the copyright footer. +html_show_copyright = False + +# some of these complain that the sections don't exist (which is not true), +# too many requests, or forbidden URL +linkcheck_ignore = [ + "http://127.0.0.1:8787/status", # Dask's diagnostics dashboard + "https://datacamp.com/community/tutorials/docstrings-python", # "forbidden" url + "https://github.com/argoproj/argo/blob/master/README.md#quickstart", + "https://console.aws.amazon.com/batch/home#/jobs", + "https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python", + "https://github.com/jazzband/pip-tools#example-usage-for-pip-compile", + "https://www.astronomer.io/docs/cloud/stable/get-started/quickstart#", + "https://eternallybored.org/misc/wget/", + "https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.from_pandas", + "https://www.oracle.com/java/technologies/javase-downloads.html", # "forbidden" url + "https://www.java.com/en/download/help/download_options.html", # "403 Client Error: Forbidden for url" + # "anchor not found" but it's a valid selector for code examples + "https://docs.delta.io/latest/delta-update.html#language-python", + "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/default_logging.yml", + "https://github.com/kedro-org/kedro/blob/main/README.md#the-humans-behind-kedro", # "anchor not found" but is valid + "https://opensource.org/license/apache2-0-php/", + "https://docs.github.com/en/rest/overview/other-authentication-methods#via-username-and-password", + "https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/api/snowflake.snowpark.DataFrameWriter.saveAsTable.html", + "https://www.educative.io/blog/advanced-yaml-syntax-cheatsheet#anchors" +] + +# retry before render a link broken (fix for "too many requests") +linkcheck_retries = 5 +linkcheck_rate_limit_timeout = 2.0 + +html_context = { + "display_github": True, + "github_url": "https://github.com/kedro-org/kedro/tree/main/docs/source", +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + +html_show_sourcelink = False + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = "Kedrodoc" + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [(master_doc, "Kedro.tex", "Kedro Documentation", "Kedro", "manual")] + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "kedro", "Kedro Documentation", [author], 1)] + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "Kedro", + "Kedro Documentation", + author, + "Kedro", + "Kedro is a Python framework for creating reproducible, maintainable and modular data science code.", + "Data-Science", + ) +] + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Kedro specific configuration ----------------------------------------- +KEDRO_MODULES = [ + "kedro.io", + "kedro.pipeline", + "kedro.runner", + "kedro.config", + "kedro.extras.datasets", + "kedro.extras.logging", + "kedro_datasets", +] + + +def get_classes(module): + importlib.import_module(module) + return [obj[0] for obj in getmembers(sys.modules[module], lambda obj: isclass(obj))] + + +def get_functions(module): + importlib.import_module(module) + return [ + obj[0] for obj in getmembers(sys.modules[module], lambda obj: isfunction(obj)) + ] + + +def remove_arrows_in_examples(lines): + for i, line in enumerate(lines): + lines[i] = line.replace(">>>", "") + + +def autolink_replacements(what: str) -> list[tuple[str, str, str]]: + """ + Create a list containing replacement tuples of the form: + (``regex``, ``replacement``, ``obj``) for all classes and methods which are + imported in ``KEDRO_MODULES`` ``__init__.py`` files. The ``replacement`` + is a reStructuredText link to their documentation. + + For example, if the docstring reads: + This LambdaDataSet loads and saves ... + + Then the word ``LambdaDataSet``, will be replaced by + :class:`~kedro.io.LambdaDataSet` + + Works for plural as well, e.g: + These ``LambdaDataSet``s load and save + + Will convert to: + These :class:`kedro.io.LambdaDataSet` load and save + + Args: + what: The objects to create replacement tuples for. Possible values + ["class", "func"]. + + Returns: + A list of tuples: (regex, replacement, obj), for all "what" objects + imported in __init__.py files of ``KEDRO_MODULES``. + + """ + replacements = [] + suggestions = [] + for module in KEDRO_MODULES: + if what == "class": + objects = get_classes(module) + elif what == "func": + objects = get_functions(module) + + # Look for recognised class names/function names which are + # surrounded by double back-ticks + if what == "class": + # first do plural only for classes + replacements += [ + ( + rf"``{obj}``s", + f":{what}:`~{module}.{obj}`\\\\s", + obj, + ) + for obj in objects + ] + + # singular + replacements += [ + (rf"``{obj}``", f":{what}:`~{module}.{obj}`", obj) for obj in objects + ] + + # Look for recognised class names/function names which are NOT + # surrounded by double back-ticks, so that we can log these in the + # terminal + if what == "class": + # first do plural only for classes + suggestions += [ + (rf"(?>>" in lines[i]: + continue + + for existing, replacement, obj in suggestions: + new = re.sub(existing, rf"{replacement}", lines[i]) + if new == lines[i]: + continue + if ":rtype:" in lines[i] or ":type " in lines[i]: + continue + + if not title_printed: + secho("-" * 50 + "\n" + name + ":\n" + "-" * 50, fg="blue") + title_printed = True + + print( + "[" + + str(i) + + "] " + + re.sub(existing, r"{}".format(style(obj, fg="magenta")), lines[i]) + ) + print( + "[" + + str(i) + + "] " + + re.sub(existing, r"``{}``".format(style(obj, fg="green")), lines[i]) + ) + + if title_printed: + print("\n") + + +def autolink_classes_and_methods(lines): + for i in range(len(lines)): + if ">>>" in lines[i]: + continue + + for existing, replacement, obj in replacements: + lines[i] = re.sub(existing, rf"{replacement}", lines[i]) + + +def autodoc_process_docstring(app, what, name, obj, options, lines): + try: + # guarded method to make sure build never fails + log_suggestions(lines, name) + autolink_classes_and_methods(lines) + except Exception as e: + print( + style( + "Failed to check for class name mentions that can be " + "converted to reStructuredText links in docstring of {}. " + "Error is: \n{}".format(name, str(e)), + fg="red", + ) + ) + + remove_arrows_in_examples(lines) + + +def env_override(default_appid): + build_version = os.getenv("READTHEDOCS_VERSION") + + if build_version == "latest": + return os.environ["HEAP_APPID_QA"] + if build_version == "stable": + return os.environ["HEAP_APPID_PROD"] + + return default_appid # default to Development for local builds + + +def _add_jinja_filters(app): + # https://github.com/crate/crate/issues/10833 + from sphinx.builders.latex import LaTeXBuilder + from sphinx.builders.linkcheck import CheckExternalLinksBuilder + + # LaTeXBuilder is used in the PDF docs build, + # and it doesn't have attribute 'templates' + if not ( + isinstance(app.builder, (LaTeXBuilder,CheckExternalLinksBuilder)) + ): + app.builder.templates.environment.filters["env_override"] = env_override + + +def _override_permalinks_icon(app): + # https://github.com/readthedocs/sphinx_rtd_theme/issues/98#issuecomment-1503211439 + app.config.html_permalinks_icon = "¶" + + +def setup(app): + app.connect("builder-inited", _add_jinja_filters) + app.connect("builder-inited", _override_permalinks_icon) + app.connect("autodoc-process-docstring", autodoc_process_docstring) + app.add_css_file("css/qb1-sphinx-rtd.css") + # fix a bug with table wraps in Read the Docs Sphinx theme: + # https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html + app.add_css_file("css/theme-overrides.css") + + +# (regex, restructuredText link replacement, object) list +replacements = [] + +# (regex, class/function name surrounded with back-ticks, object) list +suggestions = [] + +try: + # guarded code to make sure build never fails + replacements_f, suggestions_f = autolink_replacements("func") + replacements_c, suggestions_c = autolink_replacements("class") + replacements = replacements_f + replacements_c + suggestions = suggestions_f + suggestions_c +except Exception as e: + print( + style( + "Failed to create list of (regex, reStructuredText link " + "replacement) for class names and method names in docstrings. " + "Error is: \n{}".format(str(e)), + fg="red", + ) + ) + +user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0" + +myst_heading_anchors = 5 + +# https://github.com/kedro-org/kedro/issues/1772 +mermaid_output_format = "png" +# https://github.com/mermaidjs/mermaid.cli#linux-sandbox-issue +# https://github.com/mermaid-js/mermaid-cli/issues/544 +mermaid_params = ["-p", here / "puppeteer-config.json", "-s", "2"] +# https://github.com/kedro-org/kedro/issues/2451 +mermaid_version = mermaid_init_js = "" diff --git a/kedro-datasets/docs/source/index.rst b/kedro-datasets/docs/source/index.rst new file mode 100644 index 000000000..84decee2a --- /dev/null +++ b/kedro-datasets/docs/source/index.rst @@ -0,0 +1,22 @@ +.. Kedro documentation master file, created by + sphinx-quickstart on Mon Dec 18 11:31:24 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + + +API documentation +================= + +.. autosummary:: + :toctree: + :caption: API documentation + :template: autosummary/module.rst + :recursive: + + kedro_datasets + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` diff --git a/kedro-datasets/docs/source/kedro_datasets.rst b/kedro-datasets/docs/source/kedro_datasets.rst new file mode 100644 index 000000000..08147ba5e --- /dev/null +++ b/kedro-datasets/docs/source/kedro_datasets.rst @@ -0,0 +1,56 @@ +kedro_datasets +============== + +.. rubric:: Description + +.. automodule:: kedro_datasets + +.. rubric:: Classes + +.. autosummary:: + :toctree: + :template: autosummary/class.rst + + kedro_datasets.api.APIDataSet + kedro_datasets.biosequence.BioSequenceDataSet + kedro_datasets.dask.ParquetDataSet + kedro_datasets.databricks.ManagedTableDataSet + kedro_datasets.email.EmailMessageDataSet + kedro_datasets.geopandas.GeoJSONDataSet + kedro_datasets.holoviews.HoloviewsWriter + kedro_datasets.json.JSONDataSet + kedro_datasets.matplotlib.MatplotlibWriter + kedro_datasets.networkx.GMLDataSet + kedro_datasets.networkx.GraphMLDataSet + kedro_datasets.networkx.JSONDataSet + kedro_datasets.pandas.CSVDataSet + kedro_datasets.pandas.DeltaTableDataSet + kedro_datasets.pandas.ExcelDataSet + kedro_datasets.pandas.FeatherDataSet + kedro_datasets.pandas.GBQQueryDataSet + kedro_datasets.pandas.GBQTableDataSet + kedro_datasets.pandas.GenericDataSet + kedro_datasets.pandas.HDFDataSet + kedro_datasets.pandas.JSONDataSet + kedro_datasets.pandas.ParquetDataSet + kedro_datasets.pandas.SQLQueryDataSet + kedro_datasets.pandas.SQLTableDataSet + kedro_datasets.pandas.XMLDataSet + kedro_datasets.pickle.PickleDataSet + kedro_datasets.pillow.ImageDataSet + kedro_datasets.plotly.JSONDataSet + kedro_datasets.plotly.PlotlyDataSet + kedro_datasets.polars.CSVDataSet + kedro_datasets.redis.PickleDataSet + kedro_datasets.spark.DeltaTableDataSet + kedro_datasets.spark.SparkDataSet + kedro_datasets.spark.SparkHiveDataSet + kedro_datasets.spark.SparkJDBCDataSet + kedro_datasets.spark.SparkStreamingDataSet + kedro_datasets.svmlight.SVMLightDataSet + kedro_datasets.tensorflow.TensorFlowModelDataSet + kedro_datasets.text.TextDataSet + kedro_datasets.tracking.JSONDataSet + kedro_datasets.tracking.MetricsDataSet + kedro_datasets.video.VideoDataSet + kedro_datasets.yaml.YAMLDataSet diff --git a/kedro-datasets/docs/source/puppeteer-config.json b/kedro-datasets/docs/source/puppeteer-config.json new file mode 100644 index 000000000..5adc866e9 --- /dev/null +++ b/kedro-datasets/docs/source/puppeteer-config.json @@ -0,0 +1,4 @@ +{ + "args": ["--no-sandbox"], + "headless": "old" +} diff --git a/kedro-datasets/docs/source/robots.txt b/kedro-datasets/docs/source/robots.txt new file mode 100644 index 000000000..9bd9ee90d --- /dev/null +++ b/kedro-datasets/docs/source/robots.txt @@ -0,0 +1,5 @@ +User-agent: * +Disallow: * +Allow: /en/stable +Allow: /en/latest +Allow: /en/0.18.* From f838f4695eea3a41136a67e56363a4c265d68819 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 15:41:35 +0100 Subject: [PATCH 02/16] Fix setup Signed-off-by: Merel Theisen --- .github/workflows/check-plugin.yml | 21 +++++++++++++++++---- .github/workflows/run-docs-build.yml | 25 ------------------------- 2 files changed, 17 insertions(+), 29 deletions(-) delete mode 100644 .github/workflows/run-docs-build.yml diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml index 5b7fcbe87..544d266d7 100644 --- a/.github/workflows/check-plugin.yml +++ b/.github/workflows/check-plugin.yml @@ -61,10 +61,6 @@ jobs: if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' && matrix.python-version != '3.10' run: | make test-no-spark - - name: RTD build for kedro-datasets - if: inputs.plugin == 'kedro-datasets' && matrix.python-version == '3.10' - run: | - make rtd lint: defaults: @@ -97,6 +93,23 @@ jobs: - name: Run linter run: make plugin=${{ inputs.plugin }} lint + RTD-build: + if: inputs.plugin == 'kedro-datasets' && matrix.python-version == '3.10' + defaults: + run: + shell: bash + strategy: + matrix: + os: [ ubuntu-latest ] + steps: + - name: Install dependencies + run: | + cd kedro-datasets + pip install ".[docs]" + - name: RTD build for kedro-datasets + run: | + make rtd + e2e-tests: if: inputs.plugin != 'kedro-datasets' defaults: diff --git a/.github/workflows/run-docs-build.yml b/.github/workflows/run-docs-build.yml deleted file mode 100644 index 79ee833f4..000000000 --- a/.github/workflows/run-docs-build.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Run Doc builds on Kedro-Datasets - -on: - push: - branches: - - main - - try-rtd-build - paths-ignore: - - "kedro-airflow/**" - - "kedro-docker/**" - - "kedro-telemetry/**" - pull_request: - branches: - - main - - try-rtd-build - paths-ignore: - - "kedro-airflow/**" - - "kedro-docker/**" - - "kedro-telemetry/**" - -jobs: - datasets-test: - uses: ./.github/workflows/check-plugin.yml - with: - plugin: kedro-datasets From 8198bad8544e1744be114ad6689a2707c8afd543 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 15:43:21 +0100 Subject: [PATCH 03/16] Add runs-on Signed-off-by: Merel Theisen --- .github/workflows/check-plugin.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml index 544d266d7..bf974ed87 100644 --- a/.github/workflows/check-plugin.yml +++ b/.github/workflows/check-plugin.yml @@ -98,9 +98,7 @@ jobs: defaults: run: shell: bash - strategy: - matrix: - os: [ ubuntu-latest ] + runs-on: ubuntu-latest steps: - name: Install dependencies run: | From ac2c75b63ff94c57c355d64ab6d702c126da6d38 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 15:45:23 +0100 Subject: [PATCH 04/16] Fix matrix Signed-off-by: Merel Theisen --- .github/workflows/check-plugin.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml index bf974ed87..bbb14bb7f 100644 --- a/.github/workflows/check-plugin.yml +++ b/.github/workflows/check-plugin.yml @@ -94,11 +94,15 @@ jobs: run: make plugin=${{ inputs.plugin }} lint RTD-build: - if: inputs.plugin == 'kedro-datasets' && matrix.python-version == '3.10' + if: inputs.plugin == 'kedro-datasets' defaults: run: shell: bash - runs-on: ubuntu-latest + strategy: + matrix: + os: [ ubuntu-latest] + python-version: [ "3.10" ] + runs-on: ${{ matrix.os }} steps: - name: Install dependencies run: | From 4a88eadaadaf3c5c220897f92979774c99f0bda9 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 15:47:16 +0100 Subject: [PATCH 05/16] Fix cd Signed-off-by: Merel Theisen --- .github/workflows/check-plugin.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml index bbb14bb7f..d16b1010b 100644 --- a/.github/workflows/check-plugin.yml +++ b/.github/workflows/check-plugin.yml @@ -106,7 +106,7 @@ jobs: steps: - name: Install dependencies run: | - cd kedro-datasets + cd ${{ inputs.plugin }} pip install ".[docs]" - name: RTD build for kedro-datasets run: | From f2f007e5a006effd62d52dd5c4ff1c1924e6f020 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 15:50:48 +0100 Subject: [PATCH 06/16] Add more setup Signed-off-by: Merel Theisen --- .github/workflows/check-plugin.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml index d16b1010b..aeebbcc07 100644 --- a/.github/workflows/check-plugin.yml +++ b/.github/workflows/check-plugin.yml @@ -104,6 +104,18 @@ jobs: python-version: [ "3.10" ] runs-on: ${{ matrix.os }} steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{matrix.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{matrix.python-version}} + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} + restore-keys: ${{inputs.plugin}} - name: Install dependencies run: | cd ${{ inputs.plugin }} From 2b3a32be43e9a75eb7d119d5b73f720de3ecc600 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 15:53:48 +0100 Subject: [PATCH 07/16] also install test reqs Signed-off-by: Merel Theisen --- .github/workflows/check-plugin.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml index aeebbcc07..159ddfd82 100644 --- a/.github/workflows/check-plugin.yml +++ b/.github/workflows/check-plugin.yml @@ -120,6 +120,7 @@ jobs: run: | cd ${{ inputs.plugin }} pip install ".[docs]" + pip install ".[test]" - name: RTD build for kedro-datasets run: | make rtd From ec7cd00ace73f256ae89f6b320e5de494c42716f Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 19:21:22 +0100 Subject: [PATCH 08/16] Remove unnessary doc files Signed-off-by: Merel Theisen --- kedro-datasets/docs/build-docs.sh | 14 ----- kedro-datasets/docs/source/conf.py | 59 +++++++------------ .../docs/source/puppeteer-config.json | 4 -- kedro-datasets/docs/source/robots.txt | 5 -- 4 files changed, 21 insertions(+), 61 deletions(-) delete mode 100755 kedro-datasets/docs/build-docs.sh delete mode 100644 kedro-datasets/docs/source/puppeteer-config.json delete mode 100644 kedro-datasets/docs/source/robots.txt diff --git a/kedro-datasets/docs/build-docs.sh b/kedro-datasets/docs/build-docs.sh deleted file mode 100755 index d55076e11..000000000 --- a/kedro-datasets/docs/build-docs.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Exit script if you try to use an uninitialized variable. -set -o nounset - -action=$1 - -if [ "$action" == "linkcheck" ]; then - sphinx-build -WETan -j auto -D language=en -b linkcheck -d docs/build/doctrees docs/source docs/build/linkcheck -elif [ "$action" == "docs" ]; then - sphinx-build -WETa -j auto -D language=en -b html -d docs/build/doctrees docs/source docs/build/html -fi diff --git a/kedro-datasets/docs/source/conf.py b/kedro-datasets/docs/source/conf.py index ca28f16c5..042cbd1d0 100644 --- a/kedro-datasets/docs/source/conf.py +++ b/kedro-datasets/docs/source/conf.py @@ -63,7 +63,6 @@ # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] -html_static_path = ["_static"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: @@ -201,36 +200,32 @@ # html_theme_options = {"collapse_navigation": False, "style_external_links": True} -# html_extra_path used to define a path to robots.txt which is used by webcrawlers -# to ignore or allow certain links. -html_extra_path = [str(here / "robots.txt")] - # Removes, from all docs, the copyright footer. html_show_copyright = False # some of these complain that the sections don't exist (which is not true), # too many requests, or forbidden URL -linkcheck_ignore = [ - "http://127.0.0.1:8787/status", # Dask's diagnostics dashboard - "https://datacamp.com/community/tutorials/docstrings-python", # "forbidden" url - "https://github.com/argoproj/argo/blob/master/README.md#quickstart", - "https://console.aws.amazon.com/batch/home#/jobs", - "https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python", - "https://github.com/jazzband/pip-tools#example-usage-for-pip-compile", - "https://www.astronomer.io/docs/cloud/stable/get-started/quickstart#", - "https://eternallybored.org/misc/wget/", - "https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.from_pandas", - "https://www.oracle.com/java/technologies/javase-downloads.html", # "forbidden" url - "https://www.java.com/en/download/help/download_options.html", # "403 Client Error: Forbidden for url" - # "anchor not found" but it's a valid selector for code examples - "https://docs.delta.io/latest/delta-update.html#language-python", - "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/default_logging.yml", - "https://github.com/kedro-org/kedro/blob/main/README.md#the-humans-behind-kedro", # "anchor not found" but is valid - "https://opensource.org/license/apache2-0-php/", - "https://docs.github.com/en/rest/overview/other-authentication-methods#via-username-and-password", - "https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/api/snowflake.snowpark.DataFrameWriter.saveAsTable.html", - "https://www.educative.io/blog/advanced-yaml-syntax-cheatsheet#anchors" -] +# linkcheck_ignore = [ +# "http://127.0.0.1:8787/status", # Dask's diagnostics dashboard +# "https://datacamp.com/community/tutorials/docstrings-python", # "forbidden" url +# "https://github.com/argoproj/argo/blob/master/README.md#quickstart", +# "https://console.aws.amazon.com/batch/home#/jobs", +# "https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python", +# "https://github.com/jazzband/pip-tools#example-usage-for-pip-compile", +# "https://www.astronomer.io/docs/cloud/stable/get-started/quickstart#", +# "https://eternallybored.org/misc/wget/", +# "https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.from_pandas", +# "https://www.oracle.com/java/technologies/javase-downloads.html", # "forbidden" url +# "https://www.java.com/en/download/help/download_options.html", # "403 Client Error: Forbidden for url" +# # "anchor not found" but it's a valid selector for code examples +# "https://docs.delta.io/latest/delta-update.html#language-python", +# "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/default_logging.yml", +# "https://github.com/kedro-org/kedro/blob/main/README.md#the-humans-behind-kedro", # "anchor not found" but is valid +# "https://opensource.org/license/apache2-0-php/", +# "https://docs.github.com/en/rest/overview/other-authentication-methods#via-username-and-password", +# "https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/api/snowflake.snowpark.DataFrameWriter.saveAsTable.html", +# "https://www.educative.io/blog/advanced-yaml-syntax-cheatsheet#anchors" +# ] # retry before render a link broken (fix for "too many requests") linkcheck_retries = 5 @@ -517,11 +512,6 @@ def setup(app): app.connect("builder-inited", _add_jinja_filters) app.connect("builder-inited", _override_permalinks_icon) app.connect("autodoc-process-docstring", autodoc_process_docstring) - app.add_css_file("css/qb1-sphinx-rtd.css") - # fix a bug with table wraps in Read the Docs Sphinx theme: - # https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html - app.add_css_file("css/theme-overrides.css") - # (regex, restructuredText link replacement, object) list replacements = [] @@ -549,10 +539,3 @@ def setup(app): myst_heading_anchors = 5 -# https://github.com/kedro-org/kedro/issues/1772 -mermaid_output_format = "png" -# https://github.com/mermaidjs/mermaid.cli#linux-sandbox-issue -# https://github.com/mermaid-js/mermaid-cli/issues/544 -mermaid_params = ["-p", here / "puppeteer-config.json", "-s", "2"] -# https://github.com/kedro-org/kedro/issues/2451 -mermaid_version = mermaid_init_js = "" diff --git a/kedro-datasets/docs/source/puppeteer-config.json b/kedro-datasets/docs/source/puppeteer-config.json deleted file mode 100644 index 5adc866e9..000000000 --- a/kedro-datasets/docs/source/puppeteer-config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "args": ["--no-sandbox"], - "headless": "old" -} diff --git a/kedro-datasets/docs/source/robots.txt b/kedro-datasets/docs/source/robots.txt deleted file mode 100644 index 9bd9ee90d..000000000 --- a/kedro-datasets/docs/source/robots.txt +++ /dev/null @@ -1,5 +0,0 @@ -User-agent: * -Disallow: * -Allow: /en/stable -Allow: /en/latest -Allow: /en/0.18.* From fddd6db209cda965df9ed12d629ef0b597e694df Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 19:21:45 +0100 Subject: [PATCH 09/16] Remove static files Signed-off-by: Merel Theisen --- .../source/_static/css/qb1-sphinx-rtd.css | 493 ------------------ .../source/_static/css/theme-overrides.css | 35 -- 2 files changed, 528 deletions(-) delete mode 100644 kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css delete mode 100644 kedro-datasets/docs/source/_static/css/theme-overrides.css diff --git a/kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css b/kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css deleted file mode 100644 index 3f11d0cee..000000000 --- a/kedro-datasets/docs/source/_static/css/qb1-sphinx-rtd.css +++ /dev/null @@ -1,493 +0,0 @@ -@import url("https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600&display=swap"); - -html, body.wy-body-for-nav { - margin: 0; - padding: 0; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; - font-family: 'Inter', sans-serif !important; - font-weight: 400; - line-height: 2rem; -} - -html { - font-size: 62.5%; -} - -:focus { - outline-color: #006dff; -} - -body.wy-body-for-nav { - font-size: 1.4289rem; - background: rgb(250, 250, 250) !important; - color: #1e1e21; - position: relative; -} - -.wy-side-nav-search { - text-align: left; -} - -.wy-side-nav-search input[type=text] { - display: block; - box-sizing: border-box; - width: 100%; - padding: 6px 12px; - color: #666; - background-color: #fff; - font-family: 'Inter', sans-serif !important; - font-size: 1.4289rem; - border: 1px #ccc solid; - border-radius: 2px; - transition: all ease 0.15s; - box-shadow: none; -} - -.wy-side-nav-search input[type=text]:focus { - border-color: #888; - color: #333; -} - -.wy-body-for-nav .wy-nav-side { - position: fixed; - top: 0; - bottom: 0; - left: 0; - padding-bottom: 0; - width: 300px; - overflow-x: hidden; - min-height: 100%; - background: white; - z-index: 200; - box-shadow: 0 2px 5px 0 rgba(0, 0, 0, .05); -} - -.wy-body-for-nav .wy-side-scroll { - display: flex; - flex-direction: column; - width: 100%; - position: relative; - overflow-x: initial; - overflow-y: initial; -} - -.wy-body-for-nav .wy-side-nav-search { - width: 100%; - background: none; - margin: 0; - padding: 0 20px; -} -.wy-body-for-nav .wy-side-nav-search a { - display: inline-flex; - flex-direction: row-reverse; - align-items: center; - font-size: 4rem; - margin: 0; - padding: 0; - height: 7rem; - line-height: 7rem; - color: black; -} -.wy-body-for-nav .wy-side-nav-search a:before { - display: none; -} -.wy-body-for-nav .wy-side-nav-search a img.logo { - width: 4.5rem; - height: auto; - margin: 0 0.3rem 0 -0.6rem; - padding: 0; -} -.wy-body-for-nav .wy-side-nav-search>div.version { - color: #555; - display: inline-block; - margin-left: 0.5em; - font-size: 1.4rem; -} - -.wy-body-for-nav .wy-menu-vertical { - width: 300px; - margin: 0; - padding: 0px 20px 64px 20px; - overflow-y: auto; - height: 100%; -} - -.wy-body-for-nav .wy-menu-vertical p.caption { - margin: 1.5em 0 0.2em; - padding: 0; - font-size: 1.4289rem; - color: #161616; - font-weight: normal; - text-transform: none; -} - -.wy-body-for-nav .wy-menu-vertical p.caption:first-child { - margin-top: 1em; -} - -.wy-body-for-nav .wy-menu-vertical li.on a, .wy-body-for-nav .wy-menu-vertical li.current>a { - border-top: none; - border-bottom: none; -} - -.wy-body-for-nav .wy-menu-vertical li.current { - background: none; -} - -.wy-body-for-nav .wy-menu-vertical li { - margin: 0; - padding: 0 0 0 20px; -} - -.wy-body-for-nav .wy-menu-vertical li a { - display: block; - margin: 0; - padding: 0.9rem 0 !important; - font-size: 1.4289rem; - line-height: 1.2; - color: #222; - background: none !important; -} - -.wy-menu-vertical li.toctree-l1>a { - font-size: 1.4289rem; -} - -.rst-content.style-external-links a.reference.external:after { - color: inherit; - opacity: 0.8; -} - -.rst-content tt.literal, .rst-content tt.literal, .rst-content code.literal { - color: #e72000; -} - -.highlight .nv { - color: #bd269e; -} - -.highlight .c1, -.highlight .sd { - color: #147087; -} - -.highlight .nn { - color: #0e84b5; -} - -.highlight .nn { - color: #0047bf; -} - -.wy-body-for-nav .wy-menu-vertical a:hover { - color: #000; -} - -.wy-body-for-nav li span.toctree-expand { - margin-left: -20px; -} - -.wy-body-for-nav .toctree-expand:before { - font-size: 15px; - margin-right: 5px; -} - -.wy-body-for-nav .wy-nav-content-wrap { - margin-left: 300px; - background: #fafafa; -} - -.wy-body-for-nav .wy-nav-content { - padding: 0; - max-width: initial; -} - -.wy-body-for-nav .wy-breadcrumbs { - display: -webkit-box; - display: -ms-flexbox; - display: flex; - -webkit-box-align: center; - -ms-flex-align: center; - align-items: center; - width: 100%; - height: 7rem; - padding-left: 3.2rem; - background: white; - background-image: initial; - background-position-x: initial; - background-position-y: initial; - background-size: initial; - background-repeat-x: initial; - background-repeat-y: initial; - background-attachment: initial; - background-origin: initial; - background-clip: initial; - background-color: white; - box-shadow: 0 2px 5px 0 rgba(0, 0, 0, .05); -} - -.wy-body-for-nav .wy-breadcrumbs + hr { - display: none; -} -.wy-body-for-nav .wy-breadcrumbs li { - font-size: 1.8rem; -} - -.wy-body-for-nav .wy-breadcrumbs li:not(:first-child) { - margin-left: 6px; -} - -.wy-body-for-nav .wy-breadcrumbs li.wy-breadcrumbs-aside { - margin-left: 24px; -} - -.wy-body-for-nav .wy-nav-content .document { - padding: 10.2rem 3.2rem 2rem; - max-width: 100rem; -} - -.kedro-logo { - margin-bottom: 3rem; -} - -.wy-body-for-nav footer { - padding: 32px; -} - -.wy-body-for-nav .wy-body-for-nav { - background: #fafafa !important; - background-image: none; - background-size: initial; -} - -.wy-body-for-nav .wy-menu-vertical li.current a:hover { - background: none; - font-weight: 600; -} - -.wy-menu-vertical a span.toctree-expand { - color: #555 !important; -} - -.wy-menu-vertical a:hover span.toctree-expand { - color: #000 !important; -} - -.wy-body-for-nav .wy-menu-vertical li.current a { - border-right: none; - font-weight: normal; -} - -.wy-body-for-nav .wy-menu-vertical li.current a.current { - font-weight: bold; -} - -.wy-body-for-nav .toctree-l2, -.wy-body-for-nav .toctree-l3, -.wy-body-for-nav .toctree-l4, -.wy-body-for-nav .toctree-l5 { - padding-left: 8px !important; -} - -.wy-body-for-nav .toctree-l4 a { - word-break: break-word; -} - -.wy-plain-list-disc li, -.rst-content .section ul li, -.rst-content .toctree-wrapper ul li, -article ul li { - list-style: square; - margin-top: 0.35em; - margin-bottom: 0.35em; -} - -.rst-content .toctree-wrapper ul li { - margin-top: 0.5em; - margin-bottom: 0.5em; -} - -h1, h2, .rst-content .toctree-wrapper p.caption, h3, h4, h5, h6, legend { - font-family: 'Inter', sans-serif !important; - margin: 1.2em 0 1em 0; - line-height: 1.2em; - color: #272c2e; - font-weight: normal; -} - -.wy-body-for-nav .document > div > .section > *:not(:empty):first-of-type { - margin-top: 0; -} - -.wy-body-for-nav h1 { - font-size: 2.6rem; - letter-spacing: -0.3px; -} - -.wy-body-for-nav h2 { - font-size: 2.3rem; -} - -.wy-body-for-nav h3 { - font-size: 2.1rem; -} - -.wy-body-for-nav h4 { - font-size: 1.95rem; -} - -.wy-body-for-nav h5 { - font-size: 1.625rem; -} - -.wy-body-for-nav h6 { - font-size: 1.4625rem; -} - -.wy-body-for-nav p { - font-size: 100%; - margin: 0 0 1em 0; - line-height: 1.7; -} - -.wy-body-for-nav p + details { - margin: -4px 0 15px; -} - -.wy-body-for-nav blockquote { - margin: 1em 0; - padding-left: 1em; - border-left: 4px solid #ddd; - color: #6a6a6a; -} - -footer span.commit code, -footer span.commit .rst-content tt, -.rst-content footer span.commit tt { - color: #6f7070; -} - -.rst-content tt, .rst-content tt, .rst-content code { - padding: 0.16em 0.48em; -} - -.wy-body-for-nav .rst-content a, .wy-body-for-nav footer a { - font-family: 'Inter', sans-serif !important; - font-size: inherit; - color: #006ea7; - text-decoration: none; -} - -.wy-body-for-nav .rst-content a:visited, .wy-body-for-nav footer a:visited { - color: #447087; - text-decoration-color: #739eb4; -} - -.wy-body-for-nav .rst-content .btn, .wy-body-for-nav footer .btn, -.wy-body-for-nav .rst-content .btn:hover, .wy-body-for-nav footer .btn:hover, -.wy-body-for-nav .rst-content .headerlink, .wy-body-for-nav .rst-content .headerlink:hover, -.wy-body-for-nav .wy-breadcrumbs li a, -.wy-nav-content .wy-breadcrumbs .fa-github, .wy-nav-content .wy-breadcrumbs .fa-github:hover { - text-decoration: none; -} - -.wy-body-for-nav .rst-content a, .wy-body-for-nav footer a, -.wy-body-for-nav .rst-content a:hover, .wy-body-for-nav footer a:hover { - text-decoration: underline; - text-underline-position: under; - text-decoration-color: #b7d9f2; -} - -.rst-content a code span { - text-decoration: underline; - text-decoration-color: #fff; -} - -.rst-content a code { - border-bottom: 1px solid #ff8c8c; -} - -.wy-body-for-nav .wy-nav-top { - padding: 5px 20px; - background: white; - color: black; - box-shadow: 0 2px 5px 0 rgba(0, 0, 0, 0.05); -} - -.wy-body-for-nav .wy-nav-top i { - transform: translateY(6px); -} - -.wy-body-for-nav .wy-nav-top a { - font-size: 2.8rem; - color: black !important; -} - -.wy-side-scroll p.caption[role="heading"] span.caption-text { - font-weight: 700; -} - -.wy-nav-content h1, -.wy-nav-content p.caption[role="heading"] .caption-text { - font-size: 1.7857rem; - font-weight: 700; -} - - -@media screen and (max-width: 768px) { - .wy-body-for-nav .wy-nav-side { - transform: translate(-300px, 0); - transition: all ease 0.3s; - } - .wy-body-for-nav .wy-nav-side.shift { - width: 85%; - transform: translate(0, 0); - } - .wy-body-for-nav .wy-nav-content-wrap { - margin-left: 0; - transform: translate(0, 0); - transition: all ease 0.3s; - } - .wy-body-for-nav .wy-nav-content-wrap.shift { - position: relative; - left: 0; - transform: translate(85%, 0); - } - .wy-body-for-nav .wy-breadcrumbs { - display: none; - } - .wy-body-for-nav .wy-nav-content .document { - padding: 20px; - } -} - -@media screen and (min-width: 1600px) { - .wy-body-for-nav .wy-nav-side { - width: 350px; - } - .wy-body-for-nav .wy-nav-content-wrap { - margin-left: 350px; - } - html { - font-size: 70%; - } -} - -/* Fix Read The Docs side-effects */ -.wy-body-for-nav .rst-versions { - font-size: 16px; - line-height: 1; -} - -.wy-breadcrumbs { - position: fixed; - z-index: 2; -} - -/* Offset target anchor to below the fixed header */ -:target:before { - display: block; - content: ' '; - width: 1px; -} diff --git a/kedro-datasets/docs/source/_static/css/theme-overrides.css b/kedro-datasets/docs/source/_static/css/theme-overrides.css deleted file mode 100644 index bdf28d927..000000000 --- a/kedro-datasets/docs/source/_static/css/theme-overrides.css +++ /dev/null @@ -1,35 +0,0 @@ -/* override table width restrictions */ -@media screen and (min-width: 767px) { - .wy-table-responsive table td { - white-space: normal; - } - - .wy-table-responsive { - overflow: visible; - } -} - -/* override the table font-size and line-height */ -html.writer-html5 .rst-content .wy-table-responsive table.docutils th > p, -html.writer-html5 .rst-content .wy-table-responsive table.docutils td > p { - font-size: 1em; - line-height: 1em; -} - -img[alt^="mermaid-"] { - max-width: 600px; -} - -.rst-content .important { - background: #ffedcc; -} -.rst-content .important .admonition-title { - background-color: #f0b37e; -} - -/* Ensure the section title is visible when linked via a hash in the URL */ -:target:before { - content: ""; - display: block; - height: 80px; -} From 8057813d0c67bf7d1568f1eaf115fa151e1a5044 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 19:35:49 +0100 Subject: [PATCH 10/16] Remove unnecessary stuff from conf.py + lint Signed-off-by: Merel Theisen --- kedro-datasets/docs/source/conf.py | 95 ------------------------------ 1 file changed, 95 deletions(-) diff --git a/kedro-datasets/docs/source/conf.py b/kedro-datasets/docs/source/conf.py index 042cbd1d0..b4a131e88 100644 --- a/kedro-datasets/docs/source/conf.py +++ b/kedro-datasets/docs/source/conf.py @@ -21,7 +21,6 @@ from pathlib import Path from click import secho, style - from kedro import __version__ as release # -- Project information ----------------------------------------------------- @@ -50,7 +49,6 @@ "sphinx.ext.ifconfig", "sphinx.ext.viewcode", "sphinx_copybutton", - "sphinxcontrib.mermaid", "myst_parser", "notfound.extension", ] @@ -93,90 +91,22 @@ type_targets = { "py:class": ( - "object", - "bool", - "int", - "float", - "str", - "tuple", - "Any", - "Dict", - "dict", - "list", - "set", - "typing.Dict", - "typing.Iterable", - "typing.List", - "typing.Tuple", - "typing.Type", - "typing.Set", - "kedro.config.config.ConfigLoader", "kedro.io.core.AbstractDataSet", "kedro.io.AbstractDataSet", - "kedro.io.core.AbstractVersionedDataSet", - "kedro.io.core.DataSetError", "kedro.io.core.Version", - "kedro.io.data_catalog.DataCatalog", - "kedro.io.memory_dataset.MemoryDataSet", - "kedro.io.partitioned_dataset.PartitionedDataSet", - "kedro.pipeline.pipeline.Pipeline", - "kedro.runner.runner.AbstractRunner", - "kedro.runner.parallel_runner._SharedMemoryDataSet", - "kedro.runner.parallel_runner._SharedMemoryDataset", - "kedro.framework.context.context.KedroContext", - "kedro.framework.startup.ProjectMetadata", - "abc.ABC", - "Path", - "pathlib.Path", - "PurePosixPath", - "pathlib.PurePosixPath", "requests.auth.AuthBase", "google.oauth2.credentials.Credentials", - "Exception", - "CONF_SOURCE", - "integer -- return number of occurrences of value", - "integer -- return first index of value.", - "kedro.extras.datasets.pandas.json_dataset.JSONDataSet", - "kedro_datasets.pandas.json_dataset.JSONDataSet", - "pluggy._manager.PluginManager", - "PluginManager", - "_DI", - "_DO", "deltalake.table.Metadata", - # The statements below were added after subclassing UserDict in AbstractConfigLoader. - "None. Remove all items from D.", - "a shallow copy of D", - "a set-like object providing a view on D's items", - "a set-like object providing a view on D's keys", - "v, remove specified key and return the corresponding value.", - "None. Update D from dict/iterable E and F.", - "an object providing a view on D's values", - "(k, v), remove and return some (key, value) pair", - "D.get(k,d), also set D[k]=d if k not in D", - "None. Update D from mapping/iterable E and F.", "DataCatalog" ), "py:data": ( "typing.Any", - "typing.Callable", "typing.Union", "typing.Optional", "typing.Tuple", ), "py:exc": ( - "ValueError", - "BadConfigException", - "MissingConfigException", "DataSetError", - "ImportError", - "KedroCliError", - "Exception", - "TypeError", - "SyntaxError", - "CircularDependencyError", - "OutputNotUniqueError", - "ConfirmNotUniqueError", - "ParserError", "DatasetError", ), } @@ -203,30 +133,6 @@ # Removes, from all docs, the copyright footer. html_show_copyright = False -# some of these complain that the sections don't exist (which is not true), -# too many requests, or forbidden URL -# linkcheck_ignore = [ -# "http://127.0.0.1:8787/status", # Dask's diagnostics dashboard -# "https://datacamp.com/community/tutorials/docstrings-python", # "forbidden" url -# "https://github.com/argoproj/argo/blob/master/README.md#quickstart", -# "https://console.aws.amazon.com/batch/home#/jobs", -# "https://github.com/EbookFoundation/free-programming-books/blob/master/books/free-programming-books-langs.md#python", -# "https://github.com/jazzband/pip-tools#example-usage-for-pip-compile", -# "https://www.astronomer.io/docs/cloud/stable/get-started/quickstart#", -# "https://eternallybored.org/misc/wget/", -# "https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.from_pandas", -# "https://www.oracle.com/java/technologies/javase-downloads.html", # "forbidden" url -# "https://www.java.com/en/download/help/download_options.html", # "403 Client Error: Forbidden for url" -# # "anchor not found" but it's a valid selector for code examples -# "https://docs.delta.io/latest/delta-update.html#language-python", -# "https://github.com/kedro-org/kedro/blob/main/kedro/framework/project/default_logging.yml", -# "https://github.com/kedro-org/kedro/blob/main/README.md#the-humans-behind-kedro", # "anchor not found" but is valid -# "https://opensource.org/license/apache2-0-php/", -# "https://docs.github.com/en/rest/overview/other-authentication-methods#via-username-and-password", -# "https://docs.snowflake.com/en/developer-guide/snowpark/reference/python/api/snowflake.snowpark.DataFrameWriter.saveAsTable.html", -# "https://www.educative.io/blog/advanced-yaml-syntax-cheatsheet#anchors" -# ] - # retry before render a link broken (fix for "too many requests") linkcheck_retries = 5 linkcheck_rate_limit_timeout = 2.0 @@ -538,4 +444,3 @@ def setup(app): user_agent = "Mozilla/5.0 (X11; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0" myst_heading_anchors = 5 - From 0710b4ff62225600fffa478f38bfdb587209c588 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Thu, 10 Aug 2023 19:41:08 +0100 Subject: [PATCH 11/16] Add back snowparkdataset Signed-off-by: Merel Theisen --- kedro-datasets/docs/source/kedro_datasets.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/kedro-datasets/docs/source/kedro_datasets.rst b/kedro-datasets/docs/source/kedro_datasets.rst index 08147ba5e..18bff8808 100644 --- a/kedro-datasets/docs/source/kedro_datasets.rst +++ b/kedro-datasets/docs/source/kedro_datasets.rst @@ -42,6 +42,7 @@ kedro_datasets kedro_datasets.plotly.PlotlyDataSet kedro_datasets.polars.CSVDataSet kedro_datasets.redis.PickleDataSet + kedro_datasets.snowflake.SnowparkTableDataSet kedro_datasets.spark.DeltaTableDataSet kedro_datasets.spark.SparkDataSet kedro_datasets.spark.SparkHiveDataSet From 01c1b6b75c16e4a23ca1dce916e44be80261c8fe Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Fri, 11 Aug 2023 10:15:11 +0200 Subject: [PATCH 12/16] Use python 3.8 for RTD build Signed-off-by: Merel Theisen --- .github/workflows/check-plugin.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml index 159ddfd82..aa2fede26 100644 --- a/.github/workflows/check-plugin.yml +++ b/.github/workflows/check-plugin.yml @@ -101,7 +101,7 @@ jobs: strategy: matrix: os: [ ubuntu-latest] - python-version: [ "3.10" ] + python-version: [ "3.8" ] runs-on: ${{ matrix.os }} steps: - name: Checkout code From a28a3aaa4fe10f4e1e52376afbcf4bd7a6cd9ea5 Mon Sep 17 00:00:00 2001 From: Merel Theisen Date: Fri, 11 Aug 2023 10:25:14 +0200 Subject: [PATCH 13/16] Remove matrix build setup Signed-off-by: Merel Theisen --- .github/workflows/check-plugin.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml index aa2fede26..00bc8a083 100644 --- a/.github/workflows/check-plugin.yml +++ b/.github/workflows/check-plugin.yml @@ -98,23 +98,19 @@ jobs: defaults: run: shell: bash - strategy: - matrix: - os: [ ubuntu-latest] - python-version: [ "3.8" ] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python ${{matrix.python-version}} + - name: Set up Python 3.8 uses: actions/setup-python@v3 with: - python-version: ${{matrix.python-version}} + python-version: "3.8" - name: Cache python packages uses: actions/cache@v3 with: path: ~/.cache/pip - key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} + key: ${{inputs.plugin}}-ubuntu-latest-python-"3.8" restore-keys: ${{inputs.plugin}} - name: Install dependencies run: | From 72871dc6123c60b421b36d8a4661728bbad6ece5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 14 Aug 2023 17:08:55 +0200 Subject: [PATCH 14/16] Add Read the Docs configuration for kedro-datasets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- kedro-datasets/.readthedocs.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 kedro-datasets/.readthedocs.yaml diff --git a/kedro-datasets/.readthedocs.yaml b/kedro-datasets/.readthedocs.yaml new file mode 100644 index 000000000..9ded7d604 --- /dev/null +++ b/kedro-datasets/.readthedocs.yaml @@ -0,0 +1,30 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.8" + jobs: + pre_build: + - pip freeze + - python -m sphinx -WETan -j auto -D language=en -b linkcheck -d _build/doctrees docs/source _build/linkcheck + +# Build documentation in the docs/ directory with Sphinx +sphinx: + builder: html + configuration: docs/source/conf.py + fail_on_warning: true + +# Optionally set the version of Python and requirements required to build your docs +python: + install: + - method: pip + path: kedro-datasets + extra_requirements: + - docs + - test From d2d2350fe3e786304fe49a5b98a9466aec181cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 14 Aug 2023 17:11:35 +0200 Subject: [PATCH 15/16] Small fixes in Sphinx configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- kedro-datasets/docs/source/conf.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/kedro-datasets/docs/source/conf.py b/kedro-datasets/docs/source/conf.py index b4a131e88..4b231efe9 100644 --- a/kedro-datasets/docs/source/conf.py +++ b/kedro-datasets/docs/source/conf.py @@ -25,7 +25,7 @@ # -- Project information ----------------------------------------------------- -project = "kedro" +project = "kedro-datasets" author = "kedro" # The short X.Y version. @@ -139,7 +139,7 @@ html_context = { "display_github": True, - "github_url": "https://github.com/kedro-org/kedro/tree/main/docs/source", + "github_url": "https://github.com/kedro-org/kedro-plugins/tree/main/kedro-datasets/docs/source", } # Add any paths that contain custom static files (such as style sheets) here, @@ -216,12 +216,6 @@ # -- Kedro specific configuration ----------------------------------------- KEDRO_MODULES = [ - "kedro.io", - "kedro.pipeline", - "kedro.runner", - "kedro.config", - "kedro.extras.datasets", - "kedro.extras.logging", "kedro_datasets", ] From 0b432b24bb3931d02fd547660c355896f01c19a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Luis=20Cano=20Rodr=C3=ADguez?= Date: Mon, 14 Aug 2023 18:30:44 +0200 Subject: [PATCH 16/16] Fix RTD config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Luis Cano Rodríguez --- kedro-datasets/.readthedocs.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kedro-datasets/.readthedocs.yaml b/kedro-datasets/.readthedocs.yaml index 9ded7d604..ca40fa54c 100644 --- a/kedro-datasets/.readthedocs.yaml +++ b/kedro-datasets/.readthedocs.yaml @@ -12,12 +12,12 @@ build: jobs: pre_build: - pip freeze - - python -m sphinx -WETan -j auto -D language=en -b linkcheck -d _build/doctrees docs/source _build/linkcheck + - python -m sphinx -WETan -j auto -D language=en -b linkcheck -d kedro-datasets/_build/doctrees kedro-datasets/docs/source kedro-datasets/_build/linkcheck # Build documentation in the docs/ directory with Sphinx sphinx: builder: html - configuration: docs/source/conf.py + configuration: kedro-datasets/docs/source/conf.py fail_on_warning: true # Optionally set the version of Python and requirements required to build your docs