From fbddfd11d9b9bc055a3df22257be699300a435fe Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:20:56 -0400 Subject: [PATCH 01/14] Setup sphinx documentation --- docs/Makefile | 20 ++++++++++++++++++++ docs/conf.py | 25 +++++++++++++++++++++++++ docs/contributing.rst | 4 ++++ docs/getstarted.rst | 29 +++++++++++++++++++++++++++++ docs/index.rst | 19 +++++++++++++++++++ docs/make.bat | 35 +++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 7 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 docs/Makefile create mode 100644 docs/conf.py create mode 100644 docs/contributing.rst create mode 100644 docs/getstarted.rst create mode 100644 docs/index.rst create mode 100644 docs/make.bat diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d4bb2cbb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..0c7e1a9f --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,25 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'FlowCept' +copyright = '2024, Oak Ridge National Lab' +author = 'Oak Ridge National Lab' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [] + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'alabaster' +html_static_path = ['_static'] diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 00000000..d4aacd47 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1,4 @@ +Contributing +============ + +Please see the `CONTRIBUTING document `_ on GitHub for guidelines on how to contribute to the FlowCept package. diff --git a/docs/getstarted.rst b/docs/getstarted.rst new file mode 100644 index 00000000..fd4d3e09 --- /dev/null +++ b/docs/getstarted.rst @@ -0,0 +1,29 @@ +Getting Started +=============== + +Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod +tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, +quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo +consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse +cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non +proident, sunt in culpa qui officia deserunt mollit anim id est laborum. + +Installation +------------ + +Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod +tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, +quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo +consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse +cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non +proident, sunt in culpa qui officia deserunt mollit anim id est laborum. + +Usage +----- + +Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod +tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, +quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo +consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse +cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non +proident, sunt in culpa qui officia deserunt mollit anim id est laborum. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..c4099926 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,19 @@ +.. FlowCept documentation master file, created by + sphinx-quickstart on Thu Oct 24 13:41:54 2024. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +FlowCept documentation +====================== + +Add your content using ``reStructuredText`` syntax. See the +`reStructuredText `_ +documentation for details. + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + getstarted + contributing diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..32bb2452 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/pyproject.toml b/pyproject.toml index c58ce844..f0416dc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,9 +63,9 @@ responsibleai = ["torch"] tensorboard = ["tensorboard", "tensorflow", "tbparse"] dev = [ "jupyterlab", + "nbmake", "pika", "pytest", - "nbmake", "ruff", ] # Torch and some other ml-specific libs, only used for dev purposes, require the following specific versions. From 4ba23e8ee373ac4f4135cbe65ba27428e5668d4c Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:23:02 -0400 Subject: [PATCH 02/14] Add furo theme for docs --- docs/conf.py | 15 ++++++++------- docs/index.rst | 4 ++-- pyproject.toml | 1 + 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0c7e1a9f..5a555153 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,20 +6,21 @@ # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information -project = 'FlowCept' -copyright = '2024, Oak Ridge National Lab' -author = 'Oak Ridge National Lab' +project = "FlowCept" +copyright = "2024, Oak Ridge National Lab" +author = "Oak Ridge National Lab" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [] -templates_path = ['_templates'] -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'alabaster' -html_static_path = ['_static'] +html_theme = "furo" +html_title = "FlowCept" +html_static_path = ["_static"] diff --git a/docs/index.rst b/docs/index.rst index c4099926..73e22fa6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,8 +3,8 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -FlowCept documentation -====================== +FlowCept +======== Add your content using ``reStructuredText`` syntax. See the `reStructuredText `_ diff --git a/pyproject.toml b/pyproject.toml index f0416dc3..8fa2f7a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ GitHub = "https://github.com/ORNL/flowcept" [project.optional-dependencies] analytics = ["seaborn", "plotly", "scipy"] dask = ["tomli", "dask[distributed]"] +docs = ["sphinx", "furo"] kafka = ["confluent-kafka"] mlflow = ["mlflow-skinny", "SQLAlchemy", "alembic", "watchdog"] nvidia = ["nvidia-ml-py"] From 2ff8845fc2d227c240845e94ff8844c73170d33c Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:55:15 -0400 Subject: [PATCH 03/14] Add autodoc extension and api reference page --- docs/api-reference.rst | 13 +++++++++++++ docs/conf.py | 2 +- docs/index.rst | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 docs/api-reference.rst diff --git a/docs/api-reference.rst b/docs/api-reference.rst new file mode 100644 index 00000000..4ad14b45 --- /dev/null +++ b/docs/api-reference.rst @@ -0,0 +1,13 @@ +API Reference +============= + +Here. + +Core components +--------------- + +Here. + +.. error:: + + Can't import source code documentation with ``autodoc`` because flowcept requires a running database just to import the package. This needs be fixed before any API documentation can be generated by Sphinx. See `issue 144 `_ on GitHub repository for more information. diff --git a/docs/conf.py b/docs/conf.py index 5a555153..5c1558d3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,7 +13,7 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = [] +extensions = ["sphinx.ext.autodoc"] templates_path = ["_templates"] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] diff --git a/docs/index.rst b/docs/index.rst index 73e22fa6..9abed32b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,3 +17,4 @@ documentation for details. getstarted contributing + api-reference From a3b445b9349f8ed5494e3d48cee60f751ccf20cc Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Fri, 1 Nov 2024 13:07:30 -0400 Subject: [PATCH 04/14] Autodoc for Flowcept class and format README --- README.md | 51 ++++++++++-------------------------------- docs/api-reference.rst | 9 +++----- 2 files changed, 15 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 813c8a6d..579a9c95 100644 --- a/README.md +++ b/README.md @@ -6,26 +6,15 @@ # FlowCept -FlowCept is a runtime data integration system that empowers any data processing system to capture and query workflow -provenance data using data observability, requiring minimal or no changes in the target system code. It seamlessly integrates data from multiple workflows, enabling users to comprehend complex, heterogeneous, and large-scale data from various sources in federated environments. - -FlowCept is intended to address scenarios where multiple workflows in a science campaign or in an enterprise run and generate -important data to be analyzed in an integrated manner. Since these workflows may use different data manipulation tools (e.g., provenance or lineage capture tools, database systems, performance profiling tools) or can be executed within -different parallel computing systems (e.g., Dask, Spark, Workflow Management Systems), its key differentiator is the -capability to seamless and automatically integrate data from various workflows using data observability. -It builds an integrated data view at runtime enabling end-to-end exploratory data analysis and monitoring. -It follows [W3C PROV](https://www.w3.org/TR/prov-overview/) recommendations for its data schema. -It does not require changes in user codes or systems (i.e., instrumentation). All users need to do is to create adapters for their systems or tools, if one is not available yet. -In addition to observability, we provide instrumentation options for convenience. For example, by adding a `@flowcept_task` decorator on functions, FlowCept will observe their executions when they run. Also, we provide special features for PyTorch modules. Adding `@torch_task` to them will enable extra model inspection to be captured and integrated in the database at runtime. - +FlowCept is a runtime data integration system that empowers any data processing system to capture and query workflow provenance data using data observability, requiring minimal or no changes in the target system code. It seamlessly integrates data from multiple workflows, enabling users to comprehend complex, heterogeneous, and large-scale data from various sources in federated environments. + +FlowCept is intended to address scenarios where multiple workflows in a science campaign or in an enterprise run and generate important data to be analyzed in an integrated manner. Since these workflows may use different data manipulation tools (e.g., provenance or lineage capture tools, database systems, performance profiling tools) or can be executed within different parallel computing systems (e.g., Dask, Spark, Workflow Management Systems), its key differentiator is the capability to seamless and automatically integrate data from various workflows using data observability. It builds an integrated data view at runtime enabling end-to-end exploratory data analysis and monitoring. It follows [W3C PROV](https://www.w3.org/TR/prov-overview/) recommendations for its data schema. It does not require changes in user codes or systems (i.e., instrumentation). All users need to do is to create adapters for their systems or tools, if one is not available yet. In addition to observability, we provide instrumentation options for convenience. For example, by adding a `@flowcept_task` decorator on functions, FlowCept will observe their executions when they run. Also, we provide special features for PyTorch modules. Adding `@torch_task` to them will enable extra model inspection to be captured and integrated in the database at runtime. Currently, FlowCept provides adapters for: [Dask](https://www.dask.org/), [MLFlow](https://mlflow.org/), [TensorBoard](https://www.tensorflow.org/tensorboard), and [Zambeze](https://github.com/ORNL/zambeze). See the [Jupyter Notebooks](notebooks) for utilization examples. -See the [Contributing](CONTRIBUTING.md) file for guidelines to contribute with new adapters. Note that we may use the -term 'plugin' in the codebase as a synonym to adapter. Future releases should standardize the terminology to use adapter. - +See the [Contributing](CONTRIBUTING.md) file for guidelines to contribute with new adapters. Note that we may use the term 'plugin' in the codebase as a synonym to adapter. Future releases should standardize the terminology to use adapter. ## Install and Setup: @@ -33,9 +22,7 @@ term 'plugin' in the codebase as a synonym to adapter. Future releases should st `pip install .[all]` in this directory (or `pip install flowcept[all]`) if you want to install all dependencies. -For convenience, this will install all dependencies for all adapters. But it can install -dependencies for adapters you will not use. For this reason, you may want to install -like this: `pip install .[adapter_key1,adapter_key2]` for the adapters we have implemented, e.g., `pip install .[dask]`. +For convenience, this will install all dependencies for all adapters. But it can install dependencies for adapters you will not use. For this reason, you may want to install like this: `pip install .[adapter_key1,adapter_key2]` for the adapters we have implemented, e.g., `pip install .[dask]`. Currently, the optional dependencies available are: ``` @@ -48,23 +35,18 @@ pip install flowcept[analytics] # For extra analytics features. pip install flowcept[dev] # To install dev dependencies. ``` -You do not need to install any optional dependency to run Flowcept without any adapter, e.g., if you want to use simple instrumentation (see below). -In this case, you need to remove the adapter part from the [settings.yaml](resources/settings.yaml) file. +You do not need to install any optional dependency to run Flowcept without any adapter, e.g., if you want to use simple instrumentation (see below). In this case, you need to remove the adapter part from the [settings.yaml](resources/settings.yaml) file. 2. Start the Database and MQ System: To use FlowCept, one needs to start a database and a MQ system. Currently, FlowCept supports MongoDB as its database and it supports both Redis and Kafka as the MQ system. -For convenience, the default needed services can be started using a [docker-compose file](deployment/compose.yml) deployment file. -You can start them using `$> docker-compose -f deployment/compose.yml up`. +For convenience, the default needed services can be started using a [docker-compose file](deployment/compose.yml) deployment file. You can start them using `$> docker-compose -f deployment/compose.yml up`. -3. Optionally, define custom settings (e.g., routes and ports) accordingly in a settings.yaml file. There is a sample file [here](resources/sample_settings.yaml), which can be used as basis. -Then, set an environment var `FLOWCEPT_SETTINGS_PATH` with the absolute path to the yaml file. -If you do not follow this step, the default values defined [here](resources/sample_settings.yaml) will be used. +3. Optionally, define custom settings (e.g., routes and ports) accordingly in a settings.yaml file. There is a sample file [here](resources/sample_settings.yaml), which can be used as basis. Then, set an environment var `FLOWCEPT_SETTINGS_PATH` with the absolute path to the yaml file. If you do not follow this step, the default values defined [here](resources/sample_settings.yaml) will be used. 4. See the [Jupyter Notebooks](notebooks) and [Examples directory](examples) for utilization examples. - ### Simple Example with Decorators Instrumentation In addition to existing adapters to Dask, MLFlow, and others (it's extensible for any system that generates data), FlowCept also offers instrumentation via @decorators. @@ -104,9 +86,7 @@ plugin: enrich_messages: false ``` -And other variables depending on the Plugin. For instance, in Dask, timestamp creation by workers add interception overhead. -As we evolve the software, other variables that impact overhead appear and we might not stated them in this README file yet. -If you are doing extensive performance evaluation experiments using this software, please reach out to us (e.g., create an issue in the repository) for hints on how to reduce the overhead of our software. +And other variables depending on the Plugin. For instance, in Dask, timestamp creation by workers add interception overhead. As we evolve the software, other variables that impact overhead appear and we might not stated them in this README file yet. If you are doing extensive performance evaluation experiments using this software, please reach out to us (e.g., create an issue in the repository) for hints on how to reduce the overhead of our software. ## Install AMD GPU Lib @@ -129,8 +109,7 @@ Which was installed using Frontier's /opt/rocm-6.2.0/share/amd_smi ## Torch Dependencies -Some unit tests utilize `torch==2.2.2`, `torchtext=0.17.2`, and `torchvision==0.17.2`. They are only really needed to run some tests and will be installed if you run `pip install flowcept[ml_dev]` or `pip install flowcept[all]`. -If you want to use FlowCept with Torch, please adapt torch dependencies according to your project's dependencies. +Some unit tests utilize `torch==2.2.2`, `torchtext=0.17.2`, and `torchvision==0.17.2`. They are only really needed to run some tests and will be installed if you run `pip install flowcept[ml_dev]` or `pip install flowcept[all]`. If you want to use FlowCept with Torch, please adapt torch dependencies according to your project's dependencies. ## Cite us @@ -159,14 +138,8 @@ R. Souza, T. Skluzacek, S. Wilkinson, M. Ziatdinov, and R. da Silva ## Disclaimer & Get in Touch -Please note that this a research software. We encourage you to give it a try and use it with your own stack. We -are continuously working on improving documentation and adding more examples and notebooks, but we are still far from -a good documentation covering the whole system. If you are interested in working with FlowCept in your own scientific -project, we can give you a jump start if you reach out to us. Feel free to [create an issue](https://github.com/ORNL/flowcept/issues/new), -[create a new discussion thread](https://github.com/ORNL/flowcept/discussions/new/choose) or drop us an email (we trust you'll find a way to reach out to us :wink: ). +Please note that this a research software. We encourage you to give it a try and use it with your own stack. We are continuously working on improving documentation and adding more examples and notebooks, but we are still far from a good documentation covering the whole system. If you are interested in working with FlowCept in your own scientific project, we can give you a jump start if you reach out to us. Feel free to [create an issue](https://github.com/ORNL/flowcept/issues/new), [create a new discussion thread](https://github.com/ORNL/flowcept/discussions/new/choose) or drop us an email (we trust you'll find a way to reach out to us :wink: ). ## Acknowledgement -This research uses resources of the Oak Ridge Leadership Computing Facility -at the Oak Ridge National Laboratory, which is supported by the Office of -Science of the U.S. Department of Energy under Contract No. DE-AC05-00OR22725. +This research uses resources of the Oak Ridge Leadership Computing Facility at the Oak Ridge National Laboratory, which is supported by the Office of Science of the U.S. Department of Energy under Contract No. DE-AC05-00OR22725. diff --git a/docs/api-reference.rst b/docs/api-reference.rst index 4ad14b45..be6a874f 100644 --- a/docs/api-reference.rst +++ b/docs/api-reference.rst @@ -1,13 +1,10 @@ API Reference ============= -Here. +Public API documentation. Core components --------------- -Here. - -.. error:: - - Can't import source code documentation with ``autodoc`` because flowcept requires a running database just to import the package. This needs be fixed before any API documentation can be generated by Sphinx. See `issue 144 `_ on GitHub repository for more information. +.. autoclass:: flowcept.Flowcept + :members: From d52cc4513a899c97d9a136e87fc991b7627696a0 Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Fri, 1 Nov 2024 14:19:20 -0400 Subject: [PATCH 05/14] Add installation and usage content --- docs/getstarted.rst | 77 ++++++++++++++++++++++++++++++++++----------- docs/index.rst | 10 +----- 2 files changed, 60 insertions(+), 27 deletions(-) diff --git a/docs/getstarted.rst b/docs/getstarted.rst index fd4d3e09..a371b626 100644 --- a/docs/getstarted.rst +++ b/docs/getstarted.rst @@ -1,29 +1,70 @@ Getting Started =============== -Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod -tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, -quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo -consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse -cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non -proident, sunt in culpa qui officia deserunt mollit anim id est laborum. +Installation and usage instructions are detailed in the following sections. Installation ------------ -Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod -tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, -quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo -consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse -cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non -proident, sunt in culpa qui officia deserunt mollit anim id est laborum. +Installing flowcept can be accomplished by cloning the GitHub repository and installing with pip using the following terminal commands: + +.. code-block:: text + + git clone https://github.com/ORNL/flowcept.git + cd flowcept + pip install . + +Or it can be installed directly from `PyPI `_ with: + +.. code-block:: text + + pip install flowcept + +Use ``pip install flowcept[all]`` to install all dependencies for all the adapters. Alternatively, dependencies for a particular adapter can be installed; for example, ``pip install flowcept[dask]`` will install only the dependencies for the Dask adapter. The optional dependencies currently available are: + +.. code-block:: text + + pip install flowcept[mlflow] # To install mlflow's adapter + pip install flowcept[dask] # To install dask's adapter + pip install flowcept[tensorboard] # To install tensorboaard's adapter + pip install flowcept[kafka] # To utilize Kafka as the MQ, instead of Redis + pip install flowcept[nvidia] # To capture NVIDIA GPU runtime information + pip install flowcept[analytics] # For extra analytics features + pip install flowcept[dev] # To install dev dependencies + +You do not need to install any optional dependencies to run FlowCept without an adapter; for example, if you want to use simple instrumentation. In this case, you need to remove the adapter part from the settings.yaml file. Usage ----- -Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod -tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, -quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo -consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse -cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non -proident, sunt in culpa qui officia deserunt mollit anim id est laborum. +To use FlowCept, one needs to start a database and a MQ system. FlowCept currently supports MongoDB as its database and it supports both Redis and Kafka as the MQ system. For convenience, the default needed services can be started using the Docker compose deployment file from the GitHub repository: + +.. code-block:: text + + git clone https://github.com/ORNL/flowcept.git + cd flowcept + docker compose -f deployment/compose.yml up -d + +A simple example of using FlowCept without any adapters is given here: + +.. code-block:: python + + from flowcept import Flowcept, flowcept_task + + @flowcept_task + def sum_one(n): + return n + 1 + + + @flowcept_task + def mult_two(n): + return n * 2 + + + with Flowcept(workflow_name='test_workflow'): + n = 3 + o1 = sum_one(n) + o2 = mult_two(o1) + print(o2) + + print(Flowcept.db.query(filter={"workflow_id": Flowcept.current_workflow_id})) diff --git a/docs/index.rst b/docs/index.rst index 9abed32b..895e79b0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,15 +1,7 @@ -.. FlowCept documentation master file, created by - sphinx-quickstart on Thu Oct 24 13:41:54 2024. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - FlowCept ======== -Add your content using ``reStructuredText`` syntax. See the -`reStructuredText `_ -documentation for details. - +FlowCept is a runtime data integration system that empowers any data processing system to capture and query workflow provenance data using data observability, requiring minimal or no changes in the target system code. It seamlessly integrates data from multiple workflows, enabling users to comprehend complex, heterogeneous, and large-scale data from various sources in federated environments. .. toctree:: :maxdepth: 2 From d57818f3991035fd31179b35a845fad50657bf50 Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Wed, 6 Nov 2024 11:44:01 -0500 Subject: [PATCH 06/14] Add documentation section --- CONTRIBUTING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d30386ce..585b079b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -59,7 +59,6 @@ adjust the [setup.py](setup.py). 7. [Optional] Ajust flowcept.__init__.py. - # Issue Labels When a new issue is created a priority label should be added indicating how important the issue is. @@ -69,3 +68,7 @@ When a new issue is created a priority label should be added indicating how impo * `priority:high` - is essential to the completion of a milestone Reference: https://github.com/ORNL/zambeze/blob/main/CONTRIBUTING.md + +# Documentation + +[Sphinx](https://www.sphinx-doc.org) along with the [Furo theme](https://github.com/pradyunsg/furo) are used to generate documentation for the project. The **docs** optional dependencies are needed to build the documentation on your local machine. Sphinx uses docstrings from the source code to build the API documentation. These docstrings should adhere to the [NumPy docstring conventions](https://numpydoc.readthedocs.io/en/latest/format.html). From d8bf2885be0025ca39d62a2cb4d4490548ab4736 Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Wed, 6 Nov 2024 13:54:44 -0500 Subject: [PATCH 07/14] Clean up contributing guidelines --- CONTRIBUTING.md | 96 ++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 50 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 585b079b..94d45c21 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,74 +1,70 @@ +# Contributing -# Branches and Pull Requests +The following is a set of guidelines for contributing to FlowCept. Submitted code that does not conform to these guidelines will not be merged into the package. -We have two protected branches: `dev` and `main`. This means that these two branches should be as stable as -possible, especially the `main` branch. PRs to them should be peer-reviewed. +## Code Linting and Formatting -The `main` branch always has the latest working version, with a tagged release published to -[pypi](https://pypi.org/project/flowcept). -The `dev` branch may be ahead of `main` while new features are -being implemented. Feature branches should be pull requested to the `dev` branch. Pull requests into the -`main` branch should always be made from the `dev` branch and be merged when the developers agree it is time -to do so. +All Python code in the FlowCept package should adhere to the [PEP 8](https://peps.python.org/pep-0008/) style guide. All linting and formatting checks should be performed with [Ruff](https://github.com/astral-sh/ruff). Configuration for Ruff is defined in the `pyproject.toml` file. The commands shown below will run the Ruff linter and formatter checks on the source directory: -# CI/CD Pipeline +```text +ruff check src +ruff format --check src +``` -## Automated versioning +## Documentation -Flowcept ~~[attempts to]~~ follows semantic versioning. -There is a [GitHub Action](.github/workflows/create-release-n-publish.yml) that automatically bumps the -patch number of the version at PRs to the main branch and uploads to the package to pypi. +[Sphinx](https://www.sphinx-doc.org) along with the [Furo theme](https://github.com/pradyunsg/furo) are used to generate documentation for the project. The **docs** optional dependencies are needed to build the documentation on your local machine. Sphinx uses docstrings from the source code to build the API documentation. These docstrings should adhere to the [NumPy docstring conventions](https://numpydoc.readthedocs.io/en/latest/format.html). The commands shown below will build the documentation using Sphinx: -## Automated Tests and Code format check +```text +cd docs +make html +``` -All human-triggered commits to any branch will launch the [automated tests GitHub Action](.github/workflows/run-unit-tests.yml). -They will also trigger the [code format checks](.github/workflows/code-formatting.yml), -using black and flake8. So, make sure you run the following code before your commits. +## Branches and Pull Requests -```shell -$ black . -$ flake8 . -``` +There are two protected branches in this project: `dev` and `main`. This means that these two branches should be as stable as possible, especially the `main` branch. PRs to them should be peer-reviewed. -## Automated Releases +The `main` branch always has the latest working version of FlowCept, with a tagged release published to [PyPI](https://pypi.org/project/flowcept). -All commits to the `main` branch will launch the [automated publish and release GitHub Action](.github/workflows/create-release-n-publish.yml). -This will create a [tagged release](https://github.com/ORNL/flowcept/releases) and publish the package to [pypi](https://pypi.org/project/flowcept). +The `dev` branch may be ahead of `main` while new features are being implemented. Feature branches should be pull requested to the `dev` branch. Pull requests into the `main` branch should always be made from the `dev` branch and be merged when the developers agree it is time to do so. -# Checklist for Creating a new FlowCept adapter +## Issue Labels -1. Create a new package directory under `flowcept/flowceptor/plugins` -2. Create a new class that inherits from `BaseInterceptor`, and consider implementing the abstract methods: - - Observe - - Intercept - - Callback - - Prepare_task_msg - -See the existing plugins for a reference. +When a new issue is created a priority label should be added indicating how important the issue is. -3. [Optional] You may need extra classes, such as - local state manager (we provide a generic [`Interceptor State Manager`](flowcept/flowceptor/adapters/interceptor_state_manager.py)), - `@dataclasses`, Data Access Objects (`DAOs`), and event handlers. +* `priority:low` - syntactic sugar, or addressing small amounts of technical debt or non-essential features +* `priority:medium` - is important to the completion of the milestone but does not require immediate attention +* `priority:high` - is essential to the completion of a milestone -4. Create a new entry in the [settings.yaml](resources/settings.yaml) file and in the [Settings factory](flowcept/commons/settings_factory.py) +Reference: https://github.com/ORNL/zambeze/blob/main/CONTRIBUTING.md -5. Create a new `requirements.txt` file under the directory [extra_requirements](extra_requirements) and -adjust the [setup.py](setup.py). +## CI/CD Pipeline -6. [Optional] Add a new constant to [vocabulary.py](flowcept/commons/vocabulary.py). +### Automated versioning -7. [Optional] Ajust flowcept.__init__.py. +FlowCept follows semantic versioning. There is a [GitHub Action](.github/workflows/create-release-n-publish.yml) that automatically bumps the patch number of the version at PRs to the main branch and uploads the package to PyPI. -# Issue Labels +### Automated tests and code format check -When a new issue is created a priority label should be added indicating how important the issue is. +All human-triggered commits to any branch will launch the [automated tests GitHub Action](.github/workflows/run-tests.yml). They will also trigger the [code linter and formatter checks](.github/workflows/run-checks.yml), using Ruff. -* `priority:low` - syntactic sugar, or addressing small amounts of technical debt or non-essential features -* `priority:medium` - is important to the completion of the milestone but does not require immediate attention -* `priority:high` - is essential to the completion of a milestone +### Automated releases -Reference: https://github.com/ORNL/zambeze/blob/main/CONTRIBUTING.md +All commits to the `main` branch will launch the [automated publish and release GitHub Action](.github/workflows/create-release-n-publish.yml). This will create a [tagged release](https://github.com/ORNL/flowcept/releases) and publish the package to [PyPI](https://pypi.org/project/flowcept). + +## Checklist for Creating a new FlowCept adapter + +1. Create a new package directory under `flowcept/flowceptor/plugins` +2. Create a new class that inherits from `BaseInterceptor`, and consider implementing the abstract methods: + - Observe + - Intercept + - Callback + - Prepare_task_msg -# Documentation +See the existing plugins for a reference. -[Sphinx](https://www.sphinx-doc.org) along with the [Furo theme](https://github.com/pradyunsg/furo) are used to generate documentation for the project. The **docs** optional dependencies are needed to build the documentation on your local machine. Sphinx uses docstrings from the source code to build the API documentation. These docstrings should adhere to the [NumPy docstring conventions](https://numpydoc.readthedocs.io/en/latest/format.html). +3. [Optional] You may need extra classes, such as local state manager (we provide a generic [`Interceptor State Manager`](flowcept/flowceptor/adapters/interceptor_state_manager.py)), `@dataclasses`, Data Access Objects (`DAOs`), and event handlers. +4. Create a new entry in the [settings.yaml](resources/settings.yaml) file and in the [Settings factory](flowcept/commons/settings_factory.py) +5. Create a new `requirements.txt` file under the directory [extra_requirements](extra_requirements) and adjust the [setup.py](setup.py). +6. [Optional] Add a new constant to [vocabulary.py](flowcept/commons/vocabulary.py). +7. [Optional] Adjust flowcept.__init__.py. From 050f976f725f7496ad89a4ef6f364cbe8ce5ebfb Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:51:41 -0500 Subject: [PATCH 08/14] Add a Makefile to help with development commands --- Makefile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..61d7a6e5 --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +# Show help, place this first so it runs with just `make` +help: + @printf "\nCommands:\n" + @printf "\033[32mservices\033[0m run services using Docker\n" + @printf "\033[32mservices-stop\033[0m stop the running Docker services\n" + @printf "\033[32mcheck\033[0m run ruff linter and formatter checks\n" + @printf "\033[32mtest\033[0m run unit tests with pytest\n" + @printf "\033[32mclean\033[0m remove ruff and pytest cache directories\n" + +# Run services using Docker +services: + docker compose --file deployment/compose.yml up --detach + +# Stop the running Docker services and remove volumes attached to containers +services-stop: + docker compose --file deployment/compose.yml down --volumes + +# Run linter and formatter checks using ruff +check: + ruff check src + ruff format --check src + +# Run unit tests using pytest +test: + pytest --ignore=tests/decorator_tests/ml_tests/llm_tests + +# Remove cache directories generated by ruff and pytest +clean: + rm -rf .ruff_cache + rm -rf .pytest_cache From 371e6793b3e73d2672b1b67594c5fe013163daf6 Mon Sep 17 00:00:00 2001 From: Renan Francisco Santos Souza <1754978+renan-souza@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:20:10 -0500 Subject: [PATCH 09/14] Update CONTRIBUTING.md --- CONTRIBUTING.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 94d45c21..628b4028 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,10 +1,10 @@ # Contributing -The following is a set of guidelines for contributing to FlowCept. Submitted code that does not conform to these guidelines will not be merged into the package. +Welcome to the FlowCept project! To make sure new contributions align well with the project, following there are some guidelines to help you create code that fits right in. Following them increases the chances of your contributions being merged smoothly. ## Code Linting and Formatting -All Python code in the FlowCept package should adhere to the [PEP 8](https://peps.python.org/pep-0008/) style guide. All linting and formatting checks should be performed with [Ruff](https://github.com/astral-sh/ruff). Configuration for Ruff is defined in the `pyproject.toml` file. The commands shown below will run the Ruff linter and formatter checks on the source directory: +All Python code in the FlowCept package should adhere to the [PEP 8](https://peps.python.org/pep-0008/) style guide. All linting and formatting checks should be performed with [Ruff](https://github.com/astral-sh/ruff). Configuration for Ruff is defined in the [pyproject.toml](./pyproject.toml) file. The commands shown below will run the Ruff linter and formatter checks on the source directory: ```text ruff check src @@ -36,8 +36,6 @@ When a new issue is created a priority label should be added indicating how impo * `priority:medium` - is important to the completion of the milestone but does not require immediate attention * `priority:high` - is essential to the completion of a milestone -Reference: https://github.com/ORNL/zambeze/blob/main/CONTRIBUTING.md - ## CI/CD Pipeline ### Automated versioning @@ -46,7 +44,7 @@ FlowCept follows semantic versioning. There is a [GitHub Action](.github/workflo ### Automated tests and code format check -All human-triggered commits to any branch will launch the [automated tests GitHub Action](.github/workflows/run-tests.yml). They will also trigger the [code linter and formatter checks](.github/workflows/run-checks.yml), using Ruff. +All human-triggered commits to any branch will launch the [automated tests GitHub Action](.github/workflows/run-tests.yml). PRs into `dev` or `main` will also trigger the [code linter and formatter checks](.github/workflows/run-checks.yml), using Ruff. ### Automated releases @@ -65,6 +63,6 @@ See the existing plugins for a reference. 3. [Optional] You may need extra classes, such as local state manager (we provide a generic [`Interceptor State Manager`](flowcept/flowceptor/adapters/interceptor_state_manager.py)), `@dataclasses`, Data Access Objects (`DAOs`), and event handlers. 4. Create a new entry in the [settings.yaml](resources/settings.yaml) file and in the [Settings factory](flowcept/commons/settings_factory.py) -5. Create a new `requirements.txt` file under the directory [extra_requirements](extra_requirements) and adjust the [setup.py](setup.py). +5. Create a new entry in the [pyproject.toml](./pyproject.toml) file under the `[project.optional-dependencies]` section and adjust the rest of the file accordingly. 6. [Optional] Add a new constant to [vocabulary.py](flowcept/commons/vocabulary.py). 7. [Optional] Adjust flowcept.__init__.py. From 5af8d6cc095870bbe9376589b39654ade539faaf Mon Sep 17 00:00:00 2001 From: Renan Francisco Santos Souza <1754978+renan-souza@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:22:50 -0500 Subject: [PATCH 10/14] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 579a9c95..75049d8e 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ FlowCept is intended to address scenarios where multiple workflows in a science Currently, FlowCept provides adapters for: [Dask](https://www.dask.org/), [MLFlow](https://mlflow.org/), [TensorBoard](https://www.tensorflow.org/tensorboard), and [Zambeze](https://github.com/ORNL/zambeze). -See the [Jupyter Notebooks](notebooks) for utilization examples. +See the [Jupyter Notebooks](notebooks) and [Examples](examples) for utilization examples. See the [Contributing](CONTRIBUTING.md) file for guidelines to contribute with new adapters. Note that we may use the term 'plugin' in the codebase as a synonym to adapter. Future releases should standardize the terminology to use adapter. @@ -138,7 +138,7 @@ R. Souza, T. Skluzacek, S. Wilkinson, M. Ziatdinov, and R. da Silva ## Disclaimer & Get in Touch -Please note that this a research software. We encourage you to give it a try and use it with your own stack. We are continuously working on improving documentation and adding more examples and notebooks, but we are still far from a good documentation covering the whole system. If you are interested in working with FlowCept in your own scientific project, we can give you a jump start if you reach out to us. Feel free to [create an issue](https://github.com/ORNL/flowcept/issues/new), [create a new discussion thread](https://github.com/ORNL/flowcept/discussions/new/choose) or drop us an email (we trust you'll find a way to reach out to us :wink: ). +Please note that this a research software. We encourage you to give it a try and use it with your own stack. We are continuously working on improving documentation and adding more examples and notebooks, but we are still far from a good documentation covering the whole system. If you are interested in working with FlowCept in your own scientific project, we can give you a jump start if you reach out to us. Feel free to [create an issue](https://github.com/ORNL/flowcept/issues/new), [create a new discussion thread](https://github.com/ORNL/flowcept/discussions/new/choose) or drop us an email (we trust you'll find a way to reach out to us :wink:). ## Acknowledgement From 43a0c27bd2df85e88d763f6f9e0ef6b9013d8e52 Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Wed, 13 Nov 2024 11:27:45 -0500 Subject: [PATCH 11/14] Check Sphinx HTML build of docs --- .github/workflows/run-checks.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-checks.yml b/.github/workflows/run-checks.yml index 63e314bc..36a3384f 100644 --- a/.github/workflows/run-checks.yml +++ b/.github/workflows/run-checks.yml @@ -1,4 +1,4 @@ -name: Ruff linter and formatter checks +name: Linter, formatter, and docs checks on: [pull_request] permissions: @@ -18,13 +18,17 @@ jobs: python-version: "3.10" cache: "pip" - - name: Install Ruff + - name: Install package and dependencies run: | python -m pip install --upgrade pip python -m pip install ruff + python -m pip install .[docs] - name: Run ruff linter checks run: ruff check src - name: Run ruff formatter checks run: ruff format --check src + + - name: Check Sphinx HTML build + run: cd docs; make html From e275252c8e8185bdd9a732f9fd84dd3b7fc6f01a Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:40:05 -0500 Subject: [PATCH 12/14] Add sphinx-build commands to Makefile --- Makefile | 32 ++++++++++++++++++++------------ docs/Makefile | 20 -------------------- docs/make.bat | 35 ----------------------------------- 3 files changed, 20 insertions(+), 67 deletions(-) delete mode 100644 docs/Makefile delete mode 100644 docs/make.bat diff --git a/Makefile b/Makefile index 61d7a6e5..18e7b16c 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,29 @@ # Show help, place this first so it runs with just `make` help: @printf "\nCommands:\n" + @printf "\033[32mcheck\033[0m run ruff linter and formatter checks\n" + @printf "\033[32mclean\033[0m remove cache directories and Sphinx build output\n" + @printf "\033[32mdocs\033[0m build HTML documentation using Sphinx\n" @printf "\033[32mservices\033[0m run services using Docker\n" @printf "\033[32mservices-stop\033[0m stop the running Docker services\n" - @printf "\033[32mcheck\033[0m run ruff linter and formatter checks\n" @printf "\033[32mtest\033[0m run unit tests with pytest\n" - @printf "\033[32mclean\033[0m remove ruff and pytest cache directories\n" + + +# Run linter and formatter checks using ruff +check: + ruff check src + ruff format --check src + +# Remove cache directories and Sphinx build output +clean: + rm -rf .ruff_cache + rm -rf .pytest_cache + sphinx-build -M clean docs docs/_build + +# Build the HTML documentation using Sphinx +.PHONY: docs +docs: + sphinx-build -M html docs docs/_build # Run services using Docker services: @@ -15,16 +33,6 @@ services: services-stop: docker compose --file deployment/compose.yml down --volumes -# Run linter and formatter checks using ruff -check: - ruff check src - ruff format --check src - # Run unit tests using pytest test: pytest --ignore=tests/decorator_tests/ml_tests/llm_tests - -# Remove cache directories generated by ruff and pytest -clean: - rm -rf .ruff_cache - rm -rf .pytest_cache diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d4bb2cbb..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 32bb2452..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -if "%1" == "" goto help - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd From d3d3247f8c5204d55f21fd4915419daf081bac30 Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Wed, 13 Nov 2024 13:46:56 -0500 Subject: [PATCH 13/14] Use Makefile with run-checks workflow --- .github/workflows/run-checks.yml | 11 ++++------- Makefile | 8 ++++---- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/run-checks.yml b/.github/workflows/run-checks.yml index 36a3384f..cd1671cb 100644 --- a/.github/workflows/run-checks.yml +++ b/.github/workflows/run-checks.yml @@ -24,11 +24,8 @@ jobs: python -m pip install ruff python -m pip install .[docs] - - name: Run ruff linter checks - run: ruff check src + - name: Run linter and formatter checks using ruff + run: make checks - - name: Run ruff formatter checks - run: ruff format --check src - - - name: Check Sphinx HTML build - run: cd docs; make html + - name: Run HTML builder for Sphinx documentation + run: make docs diff --git a/Makefile b/Makefile index 18e7b16c..bb68820e 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,16 @@ # Show help, place this first so it runs with just `make` help: @printf "\nCommands:\n" - @printf "\033[32mcheck\033[0m run ruff linter and formatter checks\n" + @printf "\033[32mchecks\033[0m run ruff linter and formatter checks\n" @printf "\033[32mclean\033[0m remove cache directories and Sphinx build output\n" @printf "\033[32mdocs\033[0m build HTML documentation using Sphinx\n" @printf "\033[32mservices\033[0m run services using Docker\n" @printf "\033[32mservices-stop\033[0m stop the running Docker services\n" - @printf "\033[32mtest\033[0m run unit tests with pytest\n" + @printf "\033[32mtests\033[0m run unit tests with pytest\n" # Run linter and formatter checks using ruff -check: +checks: ruff check src ruff format --check src @@ -34,5 +34,5 @@ services-stop: docker compose --file deployment/compose.yml down --volumes # Run unit tests using pytest -test: +tests: pytest --ignore=tests/decorator_tests/ml_tests/llm_tests From 659cf69ded19d19b26852487fff7e5211535ced8 Mon Sep 17 00:00:00 2001 From: Gavin Wiggins <6828967+wigging@users.noreply.github.com> Date: Thu, 14 Nov 2024 14:26:35 -0500 Subject: [PATCH 14/14] Pin dask version to 2024.10.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8fa2f7a4..9a5aeb08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ GitHub = "https://github.com/ORNL/flowcept" [project.optional-dependencies] analytics = ["seaborn", "plotly", "scipy"] -dask = ["tomli", "dask[distributed]"] +dask = ["tomli", "dask[distributed]<=2024.10.0"] docs = ["sphinx", "furo"] kafka = ["confluent-kafka"] mlflow = ["mlflow-skinny", "SQLAlchemy", "alembic", "watchdog"]