From 0dce066714730620a42822620f59b25cfea18954 Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Tue, 8 Nov 2022 11:19:13 +0000 Subject: [PATCH 01/16] Update continue_config.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .circleci/continue_config.yml | 45 +++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index a035109bdb..cbfcfe7967 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -185,6 +185,24 @@ commands: - win_setup_requirements: python_version: <> + copy_kedro_datasets_main: + steps: + - run: + name: Copy kedro-datasets main to kedro/datasets + command: | + pip install --no-deps -t kedro/to_delete git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets + mv kedro/to_delete/kedro_datasets kedro/datasets + rm -r kedro/to_delete + + copy_kedro_datasets_release: + steps: + - run: + name: Copy kedro-datasets release to kedro/datasets + command: | + pip install --no-deps -t kedro/to_delete kedro-datasets + mv kedro/to_delete/kedro_datasets kedro/datasets + rm -r kedro/to_delete + jobs: e2e_tests: parameters: @@ -354,12 +372,28 @@ jobs: name: Pip-compile requirements file command: conda activate kedro_builder; make pip-compile - build_docs: + build_docs_datasets_main: + executor: + name: docker + python_version: "3.7" + steps: + - setup + - copy_kedro_datasets_main + - run: + name: Build docs + command: make build-docs + - run: + name: Pip freeze including docs dependencies + command: pip freeze + when: always + + build_docs_datasets_release: executor: name: docker python_version: "3.7" steps: - setup + - copy_kedro_datasets_main - run: name: Build docs command: make build-docs @@ -553,11 +587,11 @@ workflows: - not: <> - not: <> jobs: - - build_docs + - build_docs_datasets_main - docs_linkcheck - all_circleci_checks_succeeded: requires: - - build_docs + - build_docs_datasets_main - docs_linkcheck build_code_and_docs: @@ -596,7 +630,7 @@ workflows: matrix: parameters: python_version: ["3.7", "3.8", "3.9", "3.10"] - - build_docs + - build_docs_datasets_main - docs_linkcheck - all_circleci_checks_succeeded: requires: @@ -607,7 +641,7 @@ workflows: - lint - pip_compile - win_pip_compile - - build_docs + - build_docs_datasets_main - docs_linkcheck main_updated: @@ -699,6 +733,7 @@ workflows: - not: <> - not: <> jobs: + - build_docs_datasets_release - build_kedro: matrix: parameters: From 6459c46ee83ac4c6a4d886dcbe5756445d67cb54 Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Wed, 9 Nov 2022 13:05:09 +0000 Subject: [PATCH 02/16] Update continue_config.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .circleci/continue_config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index cbfcfe7967..9ad96aade4 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -191,7 +191,7 @@ commands: name: Copy kedro-datasets main to kedro/datasets command: | pip install --no-deps -t kedro/to_delete git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets - mv kedro/to_delete/kedro_datasets kedro/datasets + mv kedro/to_delete/kedro_datasets kedro/kedro_datasets rm -r kedro/to_delete copy_kedro_datasets_release: @@ -200,7 +200,7 @@ commands: name: Copy kedro-datasets release to kedro/datasets command: | pip install --no-deps -t kedro/to_delete kedro-datasets - mv kedro/to_delete/kedro_datasets kedro/datasets + mv kedro/to_delete/kedro_datasets kedro/kedro_datasets rm -r kedro/to_delete jobs: From 9f033b43c9cfcb370e0200ab9da43b59b0b53b6b Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Thu, 10 Nov 2022 09:42:36 +0000 Subject: [PATCH 03/16] fix autosummary issues Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .circleci/continue_config.yml | 4 ++-- docs/conf.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 9ad96aade4..cbfcfe7967 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -191,7 +191,7 @@ commands: name: Copy kedro-datasets main to kedro/datasets command: | pip install --no-deps -t kedro/to_delete git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets - mv kedro/to_delete/kedro_datasets kedro/kedro_datasets + mv kedro/to_delete/kedro_datasets kedro/datasets rm -r kedro/to_delete copy_kedro_datasets_release: @@ -200,7 +200,7 @@ commands: name: Copy kedro-datasets release to kedro/datasets command: | pip install --no-deps -t kedro/to_delete kedro-datasets - mv kedro/to_delete/kedro_datasets kedro/kedro_datasets + mv kedro/to_delete/kedro_datasets kedro/datasets rm -r kedro/to_delete jobs: diff --git a/docs/conf.py b/docs/conf.py index c917366d63..61101416e4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -333,6 +333,7 @@ "kedro.config", "kedro.extras.datasets", "kedro.extras.logging", + "kedro.datasets" ] From 4e89a54c1779a8aedd547a1858c09f29e908902d Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Thu, 10 Nov 2022 09:53:28 +0000 Subject: [PATCH 04/16] Update conf.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- docs/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 61101416e4..62ff814b09 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -140,6 +140,7 @@ "integer -- return number of occurrences of value", "integer -- return first index of value.", "kedro.extras.datasets.pandas.json_dataset.JSONDataSet", + "kedro.datasets.pandas.json_dataset.JSONDataSet", "pluggy._manager.PluginManager", "_DI", "_DO", @@ -333,7 +334,7 @@ "kedro.config", "kedro.extras.datasets", "kedro.extras.logging", - "kedro.datasets" + "kedro.datasets", ] From 336d55f26a85e3f83def6649f87b7450bf42055a Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Thu, 10 Nov 2022 09:59:57 +0000 Subject: [PATCH 05/16] Update continue_config.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .circleci/continue_config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index cbfcfe7967..f3eadf2026 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -190,7 +190,7 @@ commands: - run: name: Copy kedro-datasets main to kedro/datasets command: | - pip install --no-deps -t kedro/to_delete git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets + pip install -t kedro/to_delete git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets mv kedro/to_delete/kedro_datasets kedro/datasets rm -r kedro/to_delete @@ -199,7 +199,7 @@ commands: - run: name: Copy kedro-datasets release to kedro/datasets command: | - pip install --no-deps -t kedro/to_delete kedro-datasets + pip install -t kedro/to_delete kedro-datasets mv kedro/to_delete/kedro_datasets kedro/datasets rm -r kedro/to_delete From 75d35bf0f511b988b10b1269d5fc289062fdfb5d Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Thu, 10 Nov 2022 10:16:11 +0000 Subject: [PATCH 06/16] Update continue_config.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .circleci/continue_config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index f3eadf2026..6774fcfb51 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -190,7 +190,8 @@ commands: - run: name: Copy kedro-datasets main to kedro/datasets command: | - pip install -t kedro/to_delete git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets + pip install git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets + pip install --no-deps -t kedro/to_delete git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets mv kedro/to_delete/kedro_datasets kedro/datasets rm -r kedro/to_delete @@ -199,6 +200,7 @@ commands: - run: name: Copy kedro-datasets release to kedro/datasets command: | + pip install git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets pip install -t kedro/to_delete kedro-datasets mv kedro/to_delete/kedro_datasets kedro/datasets rm -r kedro/to_delete From cbf606d5b214f517f11986ce6d83874791794726 Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Thu, 10 Nov 2022 10:24:10 +0000 Subject: [PATCH 07/16] Update conf.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 62ff814b09..37d72112e6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -70,6 +70,7 @@ autosummary_generate = True autosummary_generate_overwrite = False napoleon_include_init_with_doc = True +autodoc_mock_imports = ["hdfs"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] From 117d6907c68bdd871986188ffb9190d96bc159df Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Mon, 14 Nov 2022 09:27:03 +0000 Subject: [PATCH 08/16] Update continue_config.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .circleci/continue_config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 6774fcfb51..33bf9ff05f 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -200,8 +200,8 @@ commands: - run: name: Copy kedro-datasets release to kedro/datasets command: | - pip install git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets - pip install -t kedro/to_delete kedro-datasets + pip install kedro-datasets + pip install --no-deps -t kedro/to_delete kedro-datasets mv kedro/to_delete/kedro_datasets kedro/datasets rm -r kedro/to_delete From b58118acb5eb81022b7d9b346598b24102ecf3bd Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Mon, 14 Nov 2022 10:39:42 +0000 Subject: [PATCH 09/16] Create kedro.datasets.rst Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- docs/source/api_docs/kedro.datasets.rst | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/source/api_docs/kedro.datasets.rst diff --git a/docs/source/api_docs/kedro.datasets.rst b/docs/source/api_docs/kedro.datasets.rst new file mode 100644 index 0000000000..28b66c09fa --- /dev/null +++ b/docs/source/api_docs/kedro.datasets.rst @@ -0,0 +1,50 @@ +kedro.datasets +===================== + +.. rubric:: Description + +.. automodule:: kedro.datasets + +.. rubric:: Classes + +.. autosummary:: + :toctree: + :template: autosummary/class.rst + + kedro.datasets.api.APIDataSet + kedro.datasets.biosequence.BioSequenceDataSet + kedro.datasets.dask.ParquetDataSet + kedro.datasets.email.EmailMessageDataSet + kedro.datasets.geopandas.GeoJSONDataSet + kedro.datasets.holoviews.HoloviewsWriter + kedro.datasets.json.JSONDataSet + kedro.datasets.matplotlib.MatplotlibWriter + kedro.datasets.networkx.GMLDataSet + kedro.datasets.networkx.GraphMLDataSet + kedro.datasets.networkx.JSONDataSet + kedro.datasets.pandas.CSVDataSet + kedro.datasets.pandas.ExcelDataSet + kedro.datasets.pandas.FeatherDataSet + kedro.datasets.pandas.GBQQueryDataSet + kedro.datasets.pandas.GBQTableDataSet + kedro.datasets.pandas.GenericDataSet + kedro.datasets.pandas.HDFDataSet + kedro.datasets.pandas.JSONDataSet + kedro.datasets.pandas.ParquetDataSet + kedro.datasets.pandas.SQLQueryDataSet + kedro.datasets.pandas.SQLTableDataSet + kedro.datasets.pandas.XMLDataSet + kedro.datasets.pickle.PickleDataSet + kedro.datasets.pillow.ImageDataSet + kedro.datasets.plotly.JSONDataSet + kedro.datasets.plotly.PlotlyDataSet + kedro.datasets.redis.PickleDataSet + kedro.datasets.spark.DeltaTableDataSet + kedro.datasets.spark.SparkDataSet + kedro.datasets.spark.SparkHiveDataSet + kedro.datasets.spark.SparkJDBCDataSet + kedro.datasets.tensorflow.TensorFlowModelDataset + kedro.datasets.text.TextDataSet + kedro.datasets.tracking.JSONDataSet + kedro.datasets.tracking.MetricsDataSet + kedro.datasets.yaml.YAMLDataSet From 2ef448003c74ca6400e8c61e1ced8e93b72e8558 Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Mon, 14 Nov 2022 11:01:35 +0000 Subject: [PATCH 10/16] Update conf.py Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- docs/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 37d72112e6..62ff814b09 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -70,7 +70,6 @@ autosummary_generate = True autosummary_generate_overwrite = False napoleon_include_init_with_doc = True -autodoc_mock_imports = ["hdfs"] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] From e023f6bc9ba36e17e8b14839734acb8966bd88df Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Mon, 14 Nov 2022 11:55:54 +0000 Subject: [PATCH 11/16] add kedro_dataset seperation to linkcheck Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .circleci/continue_config.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 33bf9ff05f..f67fe31116 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -395,7 +395,7 @@ jobs: python_version: "3.7" steps: - setup - - copy_kedro_datasets_main + - copy_kedro_datasets_release - run: name: Build docs command: make build-docs @@ -404,12 +404,28 @@ jobs: command: pip freeze when: always - docs_linkcheck: + docs_linkcheck_dataset_main: + executor: + name: docker + python_version: "3.8" + steps: + - setup + - copy_kedro_datasets_main + - run: + name: Check for broken links + command: make linkcheck + - run: + name: Pip freeze including docs dependencies + command: pip freeze + when: always + + docs_linkcheck_dataset_release: executor: name: docker python_version: "3.8" steps: - setup + - copy_kedro_datasets_release - run: name: Check for broken links command: make linkcheck From 8b8c47745e92bfb2e4a54b242b0b9fa93d68b5d0 Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Mon, 14 Nov 2022 12:05:37 +0000 Subject: [PATCH 12/16] Update continue_config.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .circleci/continue_config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index f67fe31116..33e9c146ce 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -404,7 +404,7 @@ jobs: command: pip freeze when: always - docs_linkcheck_dataset_main: + docs_linkcheck_datasets_main: executor: name: docker python_version: "3.8" @@ -419,7 +419,7 @@ jobs: command: pip freeze when: always - docs_linkcheck_dataset_release: + docs_linkcheck_datasets_release: executor: name: docker python_version: "3.8" @@ -606,11 +606,11 @@ workflows: - not: <> jobs: - build_docs_datasets_main - - docs_linkcheck + - docs_linkcheck_datasets_main - all_circleci_checks_succeeded: requires: - build_docs_datasets_main - - docs_linkcheck + - docs_linkcheck_datasets_main build_code_and_docs: when: @@ -649,7 +649,7 @@ workflows: parameters: python_version: ["3.7", "3.8", "3.9", "3.10"] - build_docs_datasets_main - - docs_linkcheck + - docs_linkcheck_datasets_main - all_circleci_checks_succeeded: requires: - e2e_tests @@ -660,7 +660,7 @@ workflows: - pip_compile - win_pip_compile - build_docs_datasets_main - - docs_linkcheck + - docs_linkcheck_datasets_main main_updated: when: From 842c73a80a55d7f2d988bfda9b0a15912201041b Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Tue, 15 Nov 2022 14:45:18 +0000 Subject: [PATCH 13/16] Update continue_config.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .circleci/continue_config.yml | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 33e9c146ce..68e1d2025b 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -571,6 +571,25 @@ jobs: sleep 120 ./tools/circleci/rtd-build.sh ${RTD_TOKEN} ${KEDRO_VERSION} + publish_kedro_datasets_RTD: + executor: + name: docker + python_version: "3.7" + steps: + - run: + name: Check Kedro version + command: | + KEDRO_VERSION=$(./tools/circleci/github_scripts/kedro_version.py ./kedro) + echo "export KEDRO_VERSION=\"${KEDRO_VERSION}\"" >> $BASH_ENV + - run: + name: Trigger Read The Docs build + command: | + ./tools/circleci/rtd-build.sh ${RTD_TOKEN} stable + # give some time for GitHub release to propagate + # otherwise RTD fails to build a new tag + sleep 120 + ./tools/circleci/rtd-build.sh ${RTD_TOKEN} ${KEDRO_VERSION} + # Trigger kedro-viz build to ensure tests in that project pass viz_build: docker: @@ -616,6 +635,7 @@ workflows: when: and: - <> + - not: <<<> - not: <> - not: <> - not: <> @@ -665,6 +685,7 @@ workflows: main_updated: when: and: + - not: <<<> - not: <> - not: <> - not: <> @@ -682,6 +703,7 @@ workflows: when: and: - <> + - not: <<<> - not: <> - not: <> jobs: @@ -695,6 +717,7 @@ workflows: when: and: - <> + - not: <<<> - not: <> - not: <> jobs: @@ -722,6 +745,7 @@ workflows: when: and: - <> + - not: <<<> - not: <> - not: <> jobs: @@ -748,6 +772,7 @@ workflows: when: and: - <> + - not: <<<> - not: <> - not: <> jobs: @@ -759,3 +784,20 @@ workflows: - publish_kedro: requires: - build_kedro + + kedro_datasets_release: + when: + and: + - <> + - not: <> + - not: <> + - not: <> + jobs: + - build_docs_datasets_release + - build_kedro: + matrix: + parameters: + python_version: [ "3.7", "3.8", "3.9", "3.10" ] + - publish_kedro_datasets_RTD: + requires: + - build_kedro From 9c8cd66618ddbc7c6eb41d16f2f65ea06503a535 Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Wed, 23 Nov 2022 16:23:13 +0000 Subject: [PATCH 14/16] option 2 use .readthedocs.yml hook Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .circleci/continue_config.yml | 115 +++------------------------------- .readthedocs.yml | 9 +++ docs/build-docs.sh | 2 + docs/kedro-datasets-docs.sh | 12 ++++ 4 files changed, 33 insertions(+), 105 deletions(-) create mode 100755 docs/kedro-datasets-docs.sh diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 68e1d2025b..a035109bdb 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -185,26 +185,6 @@ commands: - win_setup_requirements: python_version: <> - copy_kedro_datasets_main: - steps: - - run: - name: Copy kedro-datasets main to kedro/datasets - command: | - pip install git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets - pip install --no-deps -t kedro/to_delete git+https://github.com/kedro-org/kedro-plugins.git#subdirectory=kedro-datasets - mv kedro/to_delete/kedro_datasets kedro/datasets - rm -r kedro/to_delete - - copy_kedro_datasets_release: - steps: - - run: - name: Copy kedro-datasets release to kedro/datasets - command: | - pip install kedro-datasets - pip install --no-deps -t kedro/to_delete kedro-datasets - mv kedro/to_delete/kedro_datasets kedro/datasets - rm -r kedro/to_delete - jobs: e2e_tests: parameters: @@ -374,28 +354,12 @@ jobs: name: Pip-compile requirements file command: conda activate kedro_builder; make pip-compile - build_docs_datasets_main: - executor: - name: docker - python_version: "3.7" - steps: - - setup - - copy_kedro_datasets_main - - run: - name: Build docs - command: make build-docs - - run: - name: Pip freeze including docs dependencies - command: pip freeze - when: always - - build_docs_datasets_release: + build_docs: executor: name: docker python_version: "3.7" steps: - setup - - copy_kedro_datasets_release - run: name: Build docs command: make build-docs @@ -404,28 +368,12 @@ jobs: command: pip freeze when: always - docs_linkcheck_datasets_main: + docs_linkcheck: executor: name: docker python_version: "3.8" steps: - setup - - copy_kedro_datasets_main - - run: - name: Check for broken links - command: make linkcheck - - run: - name: Pip freeze including docs dependencies - command: pip freeze - when: always - - docs_linkcheck_datasets_release: - executor: - name: docker - python_version: "3.8" - steps: - - setup - - copy_kedro_datasets_release - run: name: Check for broken links command: make linkcheck @@ -571,25 +519,6 @@ jobs: sleep 120 ./tools/circleci/rtd-build.sh ${RTD_TOKEN} ${KEDRO_VERSION} - publish_kedro_datasets_RTD: - executor: - name: docker - python_version: "3.7" - steps: - - run: - name: Check Kedro version - command: | - KEDRO_VERSION=$(./tools/circleci/github_scripts/kedro_version.py ./kedro) - echo "export KEDRO_VERSION=\"${KEDRO_VERSION}\"" >> $BASH_ENV - - run: - name: Trigger Read The Docs build - command: | - ./tools/circleci/rtd-build.sh ${RTD_TOKEN} stable - # give some time for GitHub release to propagate - # otherwise RTD fails to build a new tag - sleep 120 - ./tools/circleci/rtd-build.sh ${RTD_TOKEN} ${KEDRO_VERSION} - # Trigger kedro-viz build to ensure tests in that project pass viz_build: docker: @@ -624,18 +553,17 @@ workflows: - not: <> - not: <> jobs: - - build_docs_datasets_main - - docs_linkcheck_datasets_main + - build_docs + - docs_linkcheck - all_circleci_checks_succeeded: requires: - - build_docs_datasets_main - - docs_linkcheck_datasets_main + - build_docs + - docs_linkcheck build_code_and_docs: when: and: - <> - - not: <<<> - not: <> - not: <> - not: <> @@ -668,8 +596,8 @@ workflows: matrix: parameters: python_version: ["3.7", "3.8", "3.9", "3.10"] - - build_docs_datasets_main - - docs_linkcheck_datasets_main + - build_docs + - docs_linkcheck - all_circleci_checks_succeeded: requires: - e2e_tests @@ -679,13 +607,12 @@ workflows: - lint - pip_compile - win_pip_compile - - build_docs_datasets_main - - docs_linkcheck_datasets_main + - build_docs + - docs_linkcheck main_updated: when: and: - - not: <<<> - not: <> - not: <> - not: <> @@ -703,7 +630,6 @@ workflows: when: and: - <> - - not: <<<> - not: <> - not: <> jobs: @@ -717,7 +643,6 @@ workflows: when: and: - <> - - not: <<<> - not: <> - not: <> jobs: @@ -745,7 +670,6 @@ workflows: when: and: - <> - - not: <<<> - not: <> - not: <> jobs: @@ -772,11 +696,9 @@ workflows: when: and: - <> - - not: <<<> - not: <> - not: <> jobs: - - build_docs_datasets_release - build_kedro: matrix: parameters: @@ -784,20 +706,3 @@ workflows: - publish_kedro: requires: - build_kedro - - kedro_datasets_release: - when: - and: - - <> - - not: <> - - not: <> - - not: <> - jobs: - - build_docs_datasets_release - - build_kedro: - matrix: - parameters: - python_version: [ "3.7", "3.8", "3.9", "3.10" ] - - publish_kedro_datasets_RTD: - requires: - - build_kedro diff --git a/.readthedocs.yml b/.readthedocs.yml index 507f8d271a..7d71d1ad89 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,6 +5,15 @@ # Required version: 2 +# .readthedocs.yml hook to copy kedro-datasets to kedro.datasets before the building the docs +build: + os: ubuntu-20.04 + tools: + python: "3.7" + jobs: + post_checkout: + - bash docs/kedro-datasets-docs.sh + # Build documentation in the docs/ directory with Sphinx sphinx: builder: html diff --git a/docs/build-docs.sh b/docs/build-docs.sh index b506018fab..d707d9d153 100755 --- a/docs/build-docs.sh +++ b/docs/build-docs.sh @@ -16,6 +16,8 @@ rm -rf docs/build mkdir docs/build/ cp -r docs/_templates docs/conf.py docs/*.svg docs/*.json docs/build/ +bash docs/kedro-datasets-docs.sh + if [ "$action" == "linkcheck" ]; then sphinx-build -c docs/ -WETan -j auto -D language=en -b linkcheck docs/build/ docs/build/html elif [ "$action" == "docs" ]; then diff --git a/docs/kedro-datasets-docs.sh b/docs/kedro-datasets-docs.sh new file mode 100755 index 0000000000..3945723672 --- /dev/null +++ b/docs/kedro-datasets-docs.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Script to copy kedro-datasets to kedro.datasets before the documentation build for Kedro in ReadTheDocs. + +# Exit script if you try to use an uninitialized variable. +set -o nounset + +# Exit script if a statement returns a non-true return value. +set -o errexit + +pip install --no-deps -t kedro/to_delete kedro-datasets +mv kedro/to_delete/kedro_datasets kedro/datasets +rm -r kedro/to_delete From 355026f8b9904a5ae2d17530978d64884f228f32 Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Wed, 23 Nov 2022 16:58:27 +0000 Subject: [PATCH 15/16] ci needs kedro-datasets installed Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- docs/kedro-datasets-docs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/kedro-datasets-docs.sh b/docs/kedro-datasets-docs.sh index 3945723672..10054034ab 100755 --- a/docs/kedro-datasets-docs.sh +++ b/docs/kedro-datasets-docs.sh @@ -7,6 +7,7 @@ set -o nounset # Exit script if a statement returns a non-true return value. set -o errexit +pip install kedro-datasets pip install --no-deps -t kedro/to_delete kedro-datasets mv kedro/to_delete/kedro_datasets kedro/datasets rm -r kedro/to_delete From d75a77da69f3f52955c941a9ebdab1f225e4a6aa Mon Sep 17 00:00:00 2001 From: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> Date: Thu, 24 Nov 2022 09:59:10 +0000 Subject: [PATCH 16/16] Update .readthedocs.yml Signed-off-by: SajidAlamQB <90610031+SajidAlamQB@users.noreply.github.com> --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 7d71d1ad89..5fd9cebe8a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -5,7 +5,7 @@ # Required version: 2 -# .readthedocs.yml hook to copy kedro-datasets to kedro.datasets before the building the docs +# .readthedocs.yml hook to copy kedro-datasets to kedro.datasets before building the docs build: os: ubuntu-20.04 tools: