From db9347e2f83bd3a24beee105e5a2c3052448bf10 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Fri, 24 Mar 2023 15:16:30 +0100 Subject: [PATCH 01/19] Sphinx build with recursive autosummary and custom template --- docs/api/Makefile | 20 ++++++ .../_templates/custom-class-template.rst | 34 ++++++++++ .../_templates/custom-module-template.rst | 66 +++++++++++++++++++ docs/api/source/conf.py | 42 ++++++++++++ docs/api/source/index.rst | 19 ++++++ nncf/common/api_marker.py | 8 +++ nncf/common/quantization/structs.py | 3 +- 7 files changed, 191 insertions(+), 1 deletion(-) create mode 100644 docs/api/Makefile create mode 100644 docs/api/source/_templates/custom-class-template.rst create mode 100644 docs/api/source/_templates/custom-module-template.rst create mode 100644 docs/api/source/conf.py create mode 100644 docs/api/source/index.rst create mode 100644 nncf/common/api_marker.py diff --git a/docs/api/Makefile b/docs/api/Makefile new file mode 100644 index 00000000000..d0c3cbf1020 --- /dev/null +++ b/docs/api/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/api/source/_templates/custom-class-template.rst b/docs/api/source/_templates/custom-class-template.rst new file mode 100644 index 00000000000..f73eda50ec5 --- /dev/null +++ b/docs/api/source/_templates/custom-class-template.rst @@ -0,0 +1,34 @@ +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :show-inheritance: + :inherited-members: + :special-members: __call__, __add__, __mul__ + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :nosignatures: + {% for item in methods %} + {%- if not item.startswith('_') %} + ~{{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/docs/api/source/_templates/custom-module-template.rst b/docs/api/source/_templates/custom-module-template.rst new file mode 100644 index 00000000000..d066d0e4dc2 --- /dev/null +++ b/docs/api/source/_templates/custom-module-template.rst @@ -0,0 +1,66 @@ +{{ fullname | escape | underline}} + +.. automodule:: {{ fullname }} + + {% block attributes %} + {% if attributes %} + .. rubric:: Module attributes + + .. autosummary:: + :toctree: + {% for item in attributes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + .. autosummary:: + :toctree: + :nosignatures: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: {{ _('Classes') }} + + .. autosummary:: + :toctree: + :template: custom-class-template.rst + :nosignatures: + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + :toctree: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. autosummary:: + :toctree: + :template: custom-module-template.rst + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py new file mode 100644 index 00000000000..ff804a1c9da --- /dev/null +++ b/docs/api/source/conf.py @@ -0,0 +1,42 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import os +import sys +sys.path.insert(0, os.path.abspath('../../..')) + +project = 'nncf' +copyright = '2023, Intel Corporation' +author = 'Intel Corporation' +release = 'v2.4.0' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + + +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.autosummary'] +autosummary_generate = True # Turn on sphinx.ext.autosummary + +templates_path = ['_templates'] +exclude_patterns = [] + +def skip_util_classes(app, what, name, obj, skip, options): + if hasattr(obj, 'attributes'): + attr_names = [x.name for x in obj.attributes] + if "_nncf_api_marker" in attr_names: + return skip + return True + +# def setup(sphinx): +# sphinx.connect("autodoc-skip-member", skip_util_classes) + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'alabaster' +html_static_path = ['_static'] diff --git a/docs/api/source/index.rst b/docs/api/source/index.rst new file mode 100644 index 00000000000..92e4ae084ef --- /dev/null +++ b/docs/api/source/index.rst @@ -0,0 +1,19 @@ +.. nncf documentation master file, created by + sphinx-quickstart on Fri Mar 24 11:30:33 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to nncf's documentation! +================================ + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + +.. autosummary:: + :toctree: _autosummary + :template: custom-module-template.rst + :recursive: + + nncf \ No newline at end of file diff --git a/nncf/common/api_marker.py b/nncf/common/api_marker.py new file mode 100644 index 00000000000..04d9343cc75 --- /dev/null +++ b/nncf/common/api_marker.py @@ -0,0 +1,8 @@ +class api: + API_MARKER_ATTR = "_nncf_api_marker" + + def __init__(self): + pass + + def __call__(self, obj): + setattr(obj, api.API_MARKER_ATTR, True) diff --git a/nncf/common/quantization/structs.py b/nncf/common/quantization/structs.py index 61f284b21e1..1f1d9b05835 100644 --- a/nncf/common/quantization/structs.py +++ b/nncf/common/quantization/structs.py @@ -15,6 +15,7 @@ from enum import Enum from typing import Dict, List, Optional, Any +from nncf.common.api_marker import api from nncf.common.graph import NNCFNode from nncf.common.graph import NNCFNodeName from nncf.config.schemata.defaults import QUANTIZATION_BITS @@ -167,7 +168,7 @@ def from_config(cls, qconfig: QuantizerConfig, narrow_range: bool, half_range: b narrow_range, half_range) - +@api() class QuantizationConstraints: REF_QCONF_OBJ = QuantizerConfig() From f399bc1a450fa32fa2ad605241024ea5f6d9a2af Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Fri, 24 Mar 2023 15:25:25 +0100 Subject: [PATCH 02/19] Decorator works to skip non-decorated classes --- docs/api/source/conf.py | 18 ++++++++++-------- nncf/common/api_marker.py | 1 + nncf/torch/model_creation.py | 3 ++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index ff804a1c9da..16b3861f933 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -2,7 +2,7 @@ # # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html - +import inspect # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information import os @@ -25,15 +25,17 @@ templates_path = ['_templates'] exclude_patterns = [] -def skip_util_classes(app, what, name, obj, skip, options): - if hasattr(obj, 'attributes'): - attr_names = [x.name for x in obj.attributes] - if "_nncf_api_marker" in attr_names: - return skip +module_fqn_with_api_fns_memo = set() + +def skip_non_api(app, what, name, obj, skip, options): + if what == "module": + objs = inspect.getmembers(obj) + if hasattr(obj, "_nncf_api_marker"): + return False return True -# def setup(sphinx): -# sphinx.connect("autodoc-skip-member", skip_util_classes) +def setup(sphinx): + sphinx.connect("autodoc-skip-member", skip_non_api) # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/nncf/common/api_marker.py b/nncf/common/api_marker.py index 04d9343cc75..912ee8f3ef3 100644 --- a/nncf/common/api_marker.py +++ b/nncf/common/api_marker.py @@ -6,3 +6,4 @@ def __init__(self): def __call__(self, obj): setattr(obj, api.API_MARKER_ATTR, True) + return obj diff --git a/nncf/torch/model_creation.py b/nncf/torch/model_creation.py index 62af55c5a23..2378e3d0a65 100644 --- a/nncf/torch/model_creation.py +++ b/nncf/torch/model_creation.py @@ -23,6 +23,7 @@ from torch.nn import Module from nncf.api.compression import CompressionAlgorithmController +from nncf.common.api_marker import api from nncf.common.compression import BaseCompressionAlgorithmController as BaseController from nncf.common.logging import nncf_logger from nncf.common.utils.debug import set_debug_log_dir @@ -131,7 +132,7 @@ def create_compressed_model(model: Module, synchronize_all_processes_in_distributed_mode() return compression_ctrl, compressed_model - +@api() def create_nncf_network(model: torch.nn.Module, config: NNCFConfig, dummy_forward_fn: Callable[[Module], Any] = None, From 721f86af794f86acb8c9f02cd65080b62c8009bb Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Tue, 28 Mar 2023 12:56:38 +0200 Subject: [PATCH 03/19] Recursive module search --- docs/api/source/conf.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index 16b3861f933..a0526c6c496 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -27,11 +27,39 @@ module_fqn_with_api_fns_memo = set() +_memo = {} + +def _has_api_members(module, memo): + if module in memo: + return memo[module] + modules = [] + funcs_and_classes = [] + has_api = False + for obj in inspect.getmembers(module): + if inspect.isfunction(obj) or inspect.isclass(obj): + funcs_and_classes.append(obj) + if inspect.ismodule(obj): + modules.append(obj) + for fc in funcs_and_classes: + if hasattr(fc, "_nncf_api_marker"): + has_api = True + break + for submodule in modules: + if _has_api_members(submodule, memo): + has_api = True + break + memo[module] = has_api + return has_api + def skip_non_api(app, what, name, obj, skip, options): - if what == "module": - objs = inspect.getmembers(obj) - if hasattr(obj, "_nncf_api_marker"): + if what == "module" and _has_api_members(obj, _memo): + print(f"VSHAMPOR: not skipping module {name}, has API") return False + elif hasattr(obj, "_nncf_api_marker"): + print(f"VSHAMPOR: not skipping object {name}, is API") + return False + + print(f"VSHAMPOR: skipping {what} {name}, not API") return True def setup(sphinx): From f00c02d343107a1f71864013b4b5f051cb16d158 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Mon, 3 Apr 2023 16:18:43 +0200 Subject: [PATCH 04/19] Custom API entity collector --- docs/api/collect_api_entities.py | 36 +++++++++++++++++++++++++++++ docs/api/source/conf.py | 39 ++------------------------------ 2 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 docs/api/collect_api_entities.py diff --git a/docs/api/collect_api_entities.py b/docs/api/collect_api_entities.py new file mode 100644 index 00000000000..fc541915f32 --- /dev/null +++ b/docs/api/collect_api_entities.py @@ -0,0 +1,36 @@ +import importlib +import inspect +import pkgutil + +import nncf + +modules = {} +skipped_modules = [] +for importer, modname, ispkg in pkgutil.walk_packages(path=nncf.__path__, + prefix=nncf.__name__+'.', + onerror=lambda x: None): + try: + modules[modname] = importlib.import_module(modname) + except: + skipped_modules.append(modname) + +api_fqns = [] +for modname, module in modules.items(): + print(f"{modname}") + for obj_name, obj in inspect.getmembers(module): + objects_module = getattr(obj, '__module__', None) + if objects_module == modname: + if inspect.isclass(obj) or inspect.isfunction(obj): + if hasattr(obj, "_nncf_api_marker"): + print(f"\t{obj_name}") + api_fqns.append(f"{modname}.{obj_name}") + +print() +skipped_str = '\n'.join(skipped_modules) +print(f"Skipped: {skipped_str}\n") + +print("API entities:") +for api_fqn in api_fqns: + print(api_fqn) + + diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index a0526c6c496..fbd7622d916 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -20,51 +20,16 @@ extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary'] + autosummary_generate = True # Turn on sphinx.ext.autosummary -templates_path = ['_templates'] +templates_path = ['_templates_'] exclude_patterns = [] module_fqn_with_api_fns_memo = set() _memo = {} -def _has_api_members(module, memo): - if module in memo: - return memo[module] - modules = [] - funcs_and_classes = [] - has_api = False - for obj in inspect.getmembers(module): - if inspect.isfunction(obj) or inspect.isclass(obj): - funcs_and_classes.append(obj) - if inspect.ismodule(obj): - modules.append(obj) - for fc in funcs_and_classes: - if hasattr(fc, "_nncf_api_marker"): - has_api = True - break - for submodule in modules: - if _has_api_members(submodule, memo): - has_api = True - break - memo[module] = has_api - return has_api - -def skip_non_api(app, what, name, obj, skip, options): - if what == "module" and _has_api_members(obj, _memo): - print(f"VSHAMPOR: not skipping module {name}, has API") - return False - elif hasattr(obj, "_nncf_api_marker"): - print(f"VSHAMPOR: not skipping object {name}, is API") - return False - - print(f"VSHAMPOR: skipping {what} {name}, not API") - return True - -def setup(sphinx): - sphinx.connect("autodoc-skip-member", skip_non_api) - # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output From 70613574ecc1987bb8f9c9a8884a0ab00404251b Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Mon, 3 Apr 2023 16:51:06 +0200 Subject: [PATCH 05/19] Proper rendering --- docs/api/collect_api_entities.py | 17 +++++ .../_templates/custom-class-template.rst | 34 ---------- .../_templates/custom-module-template.rst | 66 ------------------- docs/api/source/conf.py | 4 +- .../source/{index.rst => index_template.rst} | 4 +- nncf/torch/model_creation.py | 38 +++++------ 6 files changed, 39 insertions(+), 124 deletions(-) delete mode 100644 docs/api/source/_templates/custom-class-template.rst delete mode 100644 docs/api/source/_templates/custom-module-template.rst rename docs/api/source/{index.rst => index_template.rst} (85%) diff --git a/docs/api/collect_api_entities.py b/docs/api/collect_api_entities.py index fc541915f32..d98f3749f67 100644 --- a/docs/api/collect_api_entities.py +++ b/docs/api/collect_api_entities.py @@ -1,6 +1,7 @@ import importlib import inspect import pkgutil +from pathlib import Path import nncf @@ -33,4 +34,20 @@ for api_fqn in api_fqns: print(api_fqn) +DOC_ROOT = Path(__file__).parent +template_file = DOC_ROOT / 'source' / 'index_template.rst' +target_file = DOC_ROOT / 'source' / 'index.rst' +with open(template_file, encoding='utf-8', mode='r') as f: + old_lines = f.readlines() + for idx, line in enumerate(old_lines): + anchor_line = idx + if line == '.. API_ENTITIES_TEMPLATE_ANCHOR' + '\n': + break + api_section = "" + for api_fqn in api_fqns: + api_section += f" {api_fqn}\n" + content = ''.join(old_lines[:anchor_line]) + api_section + ''.join(old_lines[anchor_line + 1:]) + +with open(target_file, encoding='utf-8', mode='w+') as f: + f.write(content) diff --git a/docs/api/source/_templates/custom-class-template.rst b/docs/api/source/_templates/custom-class-template.rst deleted file mode 100644 index f73eda50ec5..00000000000 --- a/docs/api/source/_templates/custom-class-template.rst +++ /dev/null @@ -1,34 +0,0 @@ -{{ fullname | escape | underline}} - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - :members: - :show-inheritance: - :inherited-members: - :special-members: __call__, __add__, __mul__ - - {% block methods %} - {% if methods %} - .. rubric:: {{ _('Methods') }} - - .. autosummary:: - :nosignatures: - {% for item in methods %} - {%- if not item.startswith('_') %} - ~{{ name }}.{{ item }} - {%- endif -%} - {%- endfor %} - {% endif %} - {% endblock %} - - {% block attributes %} - {% if attributes %} - .. rubric:: {{ _('Attributes') }} - - .. autosummary:: - {% for item in attributes %} - ~{{ name }}.{{ item }} - {%- endfor %} - {% endif %} - {% endblock %} diff --git a/docs/api/source/_templates/custom-module-template.rst b/docs/api/source/_templates/custom-module-template.rst deleted file mode 100644 index d066d0e4dc2..00000000000 --- a/docs/api/source/_templates/custom-module-template.rst +++ /dev/null @@ -1,66 +0,0 @@ -{{ fullname | escape | underline}} - -.. automodule:: {{ fullname }} - - {% block attributes %} - {% if attributes %} - .. rubric:: Module attributes - - .. autosummary:: - :toctree: - {% for item in attributes %} - {{ item }} - {%- endfor %} - {% endif %} - {% endblock %} - - {% block functions %} - {% if functions %} - .. rubric:: {{ _('Functions') }} - - .. autosummary:: - :toctree: - :nosignatures: - {% for item in functions %} - {{ item }} - {%- endfor %} - {% endif %} - {% endblock %} - - {% block classes %} - {% if classes %} - .. rubric:: {{ _('Classes') }} - - .. autosummary:: - :toctree: - :template: custom-class-template.rst - :nosignatures: - {% for item in classes %} - {{ item }} - {%- endfor %} - {% endif %} - {% endblock %} - - {% block exceptions %} - {% if exceptions %} - .. rubric:: {{ _('Exceptions') }} - - .. autosummary:: - :toctree: - {% for item in exceptions %} - {{ item }} - {%- endfor %} - {% endif %} - {% endblock %} - -{% block modules %} -{% if modules %} -.. autosummary:: - :toctree: - :template: custom-module-template.rst - :recursive: -{% for item in modules %} - {{ item }} -{%- endfor %} -{% endif %} -{% endblock %} diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index fbd7622d916..e49f9aec27d 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -23,7 +23,7 @@ autosummary_generate = True # Turn on sphinx.ext.autosummary -templates_path = ['_templates_'] +templates_path = ['_templates'] exclude_patterns = [] module_fqn_with_api_fns_memo = set() @@ -33,5 +33,5 @@ # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'alabaster' +html_theme = 'sphinx_book_theme' html_static_path = ['_static'] diff --git a/docs/api/source/index.rst b/docs/api/source/index_template.rst similarity index 85% rename from docs/api/source/index.rst rename to docs/api/source/index_template.rst index 92e4ae084ef..01b6ddd89fe 100644 --- a/docs/api/source/index.rst +++ b/docs/api/source/index_template.rst @@ -13,7 +13,5 @@ Welcome to nncf's documentation! .. autosummary:: :toctree: _autosummary - :template: custom-module-template.rst - :recursive: - nncf \ No newline at end of file +.. API_ENTITIES_TEMPLATE_ANCHOR \ No newline at end of file diff --git a/nncf/torch/model_creation.py b/nncf/torch/model_creation.py index 2378e3d0a65..78248710f1f 100644 --- a/nncf/torch/model_creation.py +++ b/nncf/torch/model_creation.py @@ -141,29 +141,29 @@ def create_nncf_network(model: torch.nn.Module, """ The main function used to produce a model ready for adding compression from an original PyTorch model and a configuration object. - dummy_forward_fn + :param model: The original model. Should have its parameters already loaded from a checkpoint or another - source. + source. :param config: A configuration object used to determine the exact compression modifications to be applied - to the model + to the model :param dummy_forward_fn: if supplied, will be used instead of a *forward* function call to build - the internal graph representation via tracing. Specifying this is useful when the original training pipeline - has special formats of data loader output or has additional *forward* arguments other than input tensors. - Otherwise, the *forward* call of the model during graph tracing will be made with mock tensors according - to the shape specified in the config object. The dummy_forward_fn code MUST contain calls to nncf.nncf_model_input - functions made with each compressed model input tensor in the underlying model's args/kwargs tuple, and these - calls should be exactly the same as in the wrap_inputs_fn function code (see below); if dummy_forward_fn is - specified, then wrap_inputs_fn also must be specified. + the internal graph representation via tracing. Specifying this is useful when the original training pipeline + has special formats of data loader output or has additional *forward* arguments other than input tensors. + Otherwise, the *forward* call of the model during graph tracing will be made with mock tensors according + to the shape specified in the config object. The dummy_forward_fn code MUST contain calls to nncf.nncf_model_input + functions made with each compressed model input tensor in the underlying model's args/kwargs tuple, and these + calls should be exactly the same as in the wrap_inputs_fn function code (see below); if dummy_forward_fn is + specified, then wrap_inputs_fn also must be specified. :param wrap_inputs_fn: if supplied, will be used on the module's input arguments during a regular, non-dummy - forward call before passing the inputs to the underlying compressed model. This is required if the model's input - tensors that are important for compression are not supplied as arguments to the model's forward call directly, but - instead are located in a container (such as list), and the model receives the container as an argument. - wrap_inputs_fn should take as input two arguments - the tuple of positional arguments to the underlying - model's forward call, and a dict of keyword arguments to the same. The function should wrap each tensor among the - supplied model's args and kwargs that is important for compression (e.g. quantization) with an nncf.nncf_model_input - function, which is a no-operation function and marks the tensors as inputs to be traced by NNCF in the internal - graph representation. Output is the tuple of (args, kwargs), where args and kwargs are the same as were supplied in - input, but each tensor in the original input. Must be specified if dummy_forward_fn is specified. + forward call before passing the inputs to the underlying compressed model. This is required if the model's input + tensors that are important for compression are not supplied as arguments to the model's forward call directly, but + instead are located in a container (such as list), and the model receives the container as an argument. + wrap_inputs_fn should take as input two arguments - the tuple of positional arguments to the underlying + model's forward call, and a dict of keyword arguments to the same. The function should wrap each tensor among the + supplied model's args and kwargs that is important for compression (e.g. quantization) with an nncf.nncf_model_input + function, which is a no-operation function and marks the tensors as inputs to be traced by NNCF in the internal + graph representation. Output is the tuple of (args, kwargs), where args and kwargs are the same as were supplied in + input, but each tensor in the original input. Must be specified if dummy_forward_fn is specified. :param wrap_outputs_fn: if supplied, will be used on the module's output during a regular, non-dummy forward call. :return: A model wrapped by NNCFNetwork, which is ready for adding compression. """ From 5323920e0e1ca988868445dfc3997fdcb6275550 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Wed, 5 Apr 2023 13:04:50 +0200 Subject: [PATCH 06/19] Mark some entities as API --- nncf/common/initialization/dataloader.py | 3 +++ nncf/common/quantization/structs.py | 2 +- nncf/config/config.py | 2 ++ nncf/config/structures.py | 4 ++++ nncf/quantization/quantize.py | 2 ++ nncf/tensorflow/helpers/model_creation.py | 3 ++- nncf/tensorflow/initialization.py | 2 ++ nncf/tensorflow/pruning/filter_pruning/algorithm.py | 2 ++ nncf/tensorflow/quantization/algorithm.py | 2 ++ nncf/tensorflow/sparsity/magnitude/algorithm.py | 2 ++ nncf/tensorflow/sparsity/rb/algorithm.py | 2 ++ nncf/torch/initialization.py | 2 ++ nncf/torch/knowledge_distillation/algo.py | 2 ++ nncf/torch/model_creation.py | 3 ++- nncf/torch/pruning/filter_pruning/algo.py | 2 ++ nncf/torch/quantization/algo.py | 2 ++ nncf/torch/sparsity/base_algo.py | 2 ++ nncf/torch/sparsity/const/algo.py | 2 ++ nncf/torch/sparsity/magnitude/algo.py | 2 ++ nncf/torch/sparsity/rb/algo.py | 2 ++ nncf/torch/structures.py | 6 ++++++ 21 files changed, 48 insertions(+), 3 deletions(-) diff --git a/nncf/common/initialization/dataloader.py b/nncf/common/initialization/dataloader.py index a91b8ca6336..c7011b8086c 100644 --- a/nncf/common/initialization/dataloader.py +++ b/nncf/common/initialization/dataloader.py @@ -13,7 +13,10 @@ from abc import ABC, abstractmethod +from nncf.common.api_marker import api + +@api() class NNCFDataLoader(ABC): """ Wraps a custom data source. diff --git a/nncf/common/quantization/structs.py b/nncf/common/quantization/structs.py index 1f1d9b05835..251991b2d46 100644 --- a/nncf/common/quantization/structs.py +++ b/nncf/common/quantization/structs.py @@ -168,7 +168,7 @@ def from_config(cls, qconfig: QuantizerConfig, narrow_range: bool, half_range: b narrow_range, half_range) -@api() + class QuantizationConstraints: REF_QCONF_OBJ = QuantizerConfig() diff --git a/nncf/config/config.py b/nncf/config/config.py index ff22768de24..68a8d3ffae7 100644 --- a/nncf/config/config.py +++ b/nncf/config/config.py @@ -20,6 +20,7 @@ import jsonschema import jstyleson as json +from nncf.common.api_marker import api from nncf.common.logging import nncf_logger from nncf.common.utils.os import safe_open from nncf.config.definitions import SCHEMA_VISUALIZATION_URL @@ -29,6 +30,7 @@ from nncf.config.structures import NNCFExtraConfigStruct +@api() class NNCFConfig(dict): """A regular dictionary object extended with some utility functions.""" diff --git a/nncf/config/structures.py b/nncf/config/structures.py index a7df6c9569d..319c60a2cb4 100644 --- a/nncf/config/structures.py +++ b/nncf/config/structures.py @@ -13,6 +13,7 @@ from typing import Optional, Callable +from nncf.common.api_marker import api from nncf.common.initialization.dataloader import NNCFDataLoader @@ -27,6 +28,7 @@ def get_id(cls) -> str: raise NotImplementedError +@api() class QuantizationRangeInitArgs(NNCFExtraConfigStruct): """ Stores additional arguments for quantization range initialization algorithms. @@ -59,6 +61,7 @@ def get_id(cls) -> str: return 'quantization_range_init_args' +@api() class BNAdaptationInitArgs(NNCFExtraConfigStruct): """ Stores additional arguments for batchnorm statistics adaptation algorithm. @@ -91,6 +94,7 @@ def get_id(cls) -> str: return 'bn_adaptation_init_args' +@api() class ModelEvaluationArgs(NNCFExtraConfigStruct): def __init__(self, eval_fn: Callable): diff --git a/nncf/quantization/quantize.py b/nncf/quantization/quantize.py index d6f687df4af..c3e95ac0870 100644 --- a/nncf/quantization/quantize.py +++ b/nncf/quantization/quantize.py @@ -17,6 +17,7 @@ from typing import Any from nncf.api.compression import TModel +from nncf.common.api_marker import api from nncf.common.quantization.structs import QuantizationPreset from nncf.common.utils.backend import BackendType from nncf.common.utils.backend import get_backend @@ -26,6 +27,7 @@ from nncf.parameters import TargetDevice +@api() def quantize(model: TModel, calibration_dataset: Dataset, preset: QuantizationPreset = QuantizationPreset.PERFORMANCE, diff --git a/nncf/tensorflow/helpers/model_creation.py b/nncf/tensorflow/helpers/model_creation.py index 88813d8edd7..2322d18c0ae 100644 --- a/nncf/tensorflow/helpers/model_creation.py +++ b/nncf/tensorflow/helpers/model_creation.py @@ -21,6 +21,7 @@ from nncf import NNCFConfig from nncf.api.compression import CompressionAlgorithmController +from nncf.common.api_marker import api from nncf.common.compression import BaseCompressionAlgorithmController as BaseController from nncf.config.extractors import extract_algorithm_names from nncf.config.telemetry_extractors import CompressionStartedFromConfig @@ -58,7 +59,7 @@ def create_compression_algorithm_builder(config: NNCFConfig, return TFCompositeCompressionAlgorithmBuilder(config, should_init) - +@api() @tracked_function(NNCF_TF_CATEGORY, [CompressionStartedFromConfig(argname="config"), ]) def create_compressed_model(model: tf.keras.Model, config: NNCFConfig, diff --git a/nncf/tensorflow/initialization.py b/nncf/tensorflow/initialization.py index bdd67fe5671..7f6a89224e9 100644 --- a/nncf/tensorflow/initialization.py +++ b/nncf/tensorflow/initialization.py @@ -13,6 +13,7 @@ import tensorflow as tf +from nncf.common.api_marker import api from nncf.common.initialization.dataloader import NNCFDataLoader from nncf.config import NNCFConfig from nncf.config.structures import BNAdaptationInitArgs @@ -38,6 +39,7 @@ def __iter__(self): return iter(self._data_loader) +@api() def register_default_init_args(nncf_config: NNCFConfig, data_loader: tf.data.Dataset, batch_size: int, diff --git a/nncf/tensorflow/pruning/filter_pruning/algorithm.py b/nncf/tensorflow/pruning/filter_pruning/algorithm.py index a0f8da72d76..a03fd24e8da 100644 --- a/nncf/tensorflow/pruning/filter_pruning/algorithm.py +++ b/nncf/tensorflow/pruning/filter_pruning/algorithm.py @@ -20,6 +20,7 @@ from nncf import NNCFConfig from nncf.api.compression import CompressionLoss from nncf.api.compression import CompressionStage +from nncf.common.api_marker import api from nncf.common.graph import NNCFGraph from nncf.common.initialization.batchnorm_adaptation import BatchnormAdaptationAlgorithm from nncf.common.pruning.clusterization import Cluster @@ -97,6 +98,7 @@ def _get_types_of_grouping_ops(self) -> List[str]: return TFElementwisePruningOp.get_all_op_aliases() +@api() @ADAPTIVE_COMPRESSION_CONTROLLERS.register('tf_filter_pruning') class FilterPruningController(BasePruningAlgoController): """ diff --git a/nncf/tensorflow/quantization/algorithm.py b/nncf/tensorflow/quantization/algorithm.py index 8819bb5a3ac..475aa4409b2 100644 --- a/nncf/tensorflow/quantization/algorithm.py +++ b/nncf/tensorflow/quantization/algorithm.py @@ -22,6 +22,7 @@ from nncf.api.compression import CompressionLoss from nncf.api.compression import CompressionScheduler from nncf.api.compression import CompressionStage +from nncf.common.api_marker import api from nncf.common.compression import BaseCompressionAlgorithmController from nncf.common.graph import INPUT_NOOP_METATYPES from nncf.common.graph import OUTPUT_NOOP_METATYPES @@ -680,6 +681,7 @@ def _get_quantizer_operation_name(self, layer_name, weight_attr_name): return f'{layer_name}_{weight_attr_name}_quantizer' +@api() class QuantizationController(BaseCompressionAlgorithmController): def __init__(self, target_model, config, op_names: List[str]): super().__init__(target_model) diff --git a/nncf/tensorflow/sparsity/magnitude/algorithm.py b/nncf/tensorflow/sparsity/magnitude/algorithm.py index 5354168b41b..b2c6a3e772f 100644 --- a/nncf/tensorflow/sparsity/magnitude/algorithm.py +++ b/nncf/tensorflow/sparsity/magnitude/algorithm.py @@ -20,6 +20,7 @@ from nncf.api.compression import CompressionScheduler from nncf.api.compression import CompressionStage from nncf.common.accuracy_aware_training.training_loop import ADAPTIVE_COMPRESSION_CONTROLLERS +from nncf.common.api_marker import api from nncf.common.graph import OUTPUT_NOOP_METATYPES from nncf.common.graph.transformations.commands import TransformationPriority from nncf.common.initialization.batchnorm_adaptation import BatchnormAdaptationAlgorithm @@ -132,6 +133,7 @@ def initialize(self, model: tf.keras.Model) -> None: pass +@api() @ADAPTIVE_COMPRESSION_CONTROLLERS.register('tf_magnitude_sparsity') class MagnitudeSparsityController(BaseSparsityController): """ diff --git a/nncf/tensorflow/sparsity/rb/algorithm.py b/nncf/tensorflow/sparsity/rb/algorithm.py index 737ec108365..80689be691f 100644 --- a/nncf/tensorflow/sparsity/rb/algorithm.py +++ b/nncf/tensorflow/sparsity/rb/algorithm.py @@ -19,6 +19,7 @@ from nncf import NNCFConfig from nncf.common.accuracy_aware_training.training_loop import ADAPTIVE_COMPRESSION_CONTROLLERS +from nncf.common.api_marker import api from nncf.common.graph.transformations.commands import TransformationPriority from nncf.common.schedulers import StubCompressionScheduler from nncf.common.scopes import check_scopes_in_graph @@ -107,6 +108,7 @@ def initialize(self, model: tf.keras.Model) -> None: pass +@api() @ADAPTIVE_COMPRESSION_CONTROLLERS.register('tf_rb_sparsity') class RBSparsityController(BaseSparsityController): def __init__(self, target_model, config: NNCFConfig, op_names: List[str]): diff --git a/nncf/torch/initialization.py b/nncf/torch/initialization.py index 58e3c25d128..42bbebcb736 100644 --- a/nncf/torch/initialization.py +++ b/nncf/torch/initialization.py @@ -11,6 +11,7 @@ from torch.nn.modules.loss import _Loss from torch.utils.data import DataLoader +from nncf.common.api_marker import api from nncf.common.initialization.dataloader import NNCFDataLoader from nncf.common.logging import nncf_logger from nncf.common.logging.progress_bar import ProgressBar @@ -224,6 +225,7 @@ def default_criterion_fn(outputs: Any, target: Any, criterion: Any) -> torch.Ten return criterion(outputs, target) +@api() def register_default_init_args(nncf_config: 'NNCFConfig', train_loader: torch.utils.data.DataLoader, criterion: _Loss = None, diff --git a/nncf/torch/knowledge_distillation/algo.py b/nncf/torch/knowledge_distillation/algo.py index 46fd309be7d..6940a9b6672 100644 --- a/nncf/torch/knowledge_distillation/algo.py +++ b/nncf/torch/knowledge_distillation/algo.py @@ -15,6 +15,7 @@ from torch import nn +from nncf.common.api_marker import api from nncf.common.schedulers import BaseCompressionScheduler from nncf.common.statistics import NNCFStatistics from nncf.config.schemata.defaults import KNOWLEDGE_DISTILLATION_SCALE @@ -52,6 +53,7 @@ def initialize(self, model: NNCFNetwork) -> None: pass +@api() class KnowledgeDistillationController(PTCompressionAlgorithmController): def __init__(self, target_model: NNCFNetwork, original_model: nn.Module, kd_type: str, scale: float, temperature: float): diff --git a/nncf/torch/model_creation.py b/nncf/torch/model_creation.py index 78248710f1f..cff847499e8 100644 --- a/nncf/torch/model_creation.py +++ b/nncf/torch/model_creation.py @@ -45,6 +45,7 @@ from nncf.torch.utils import training_mode_switcher +@api() @tracked_function(NNCF_PT_CATEGORY, [CompressionStartedFromConfig(argname="config"), ]) def create_compressed_model(model: Module, config: NNCFConfig, @@ -132,7 +133,7 @@ def create_compressed_model(model: Module, synchronize_all_processes_in_distributed_mode() return compression_ctrl, compressed_model -@api() + def create_nncf_network(model: torch.nn.Module, config: NNCFConfig, dummy_forward_fn: Callable[[Module], Any] = None, diff --git a/nncf/torch/pruning/filter_pruning/algo.py b/nncf/torch/pruning/filter_pruning/algo.py index 6449a55b798..be2301f6cfd 100644 --- a/nncf/torch/pruning/filter_pruning/algo.py +++ b/nncf/torch/pruning/filter_pruning/algo.py @@ -26,6 +26,7 @@ from nncf.api.compression import CompressionLoss from nncf.api.compression import CompressionStage from nncf.common.accuracy_aware_training.training_loop import ADAPTIVE_COMPRESSION_CONTROLLERS +from nncf.common.api_marker import api from nncf.common.graph import NNCFNode from nncf.common.graph import NNCFNodeName from nncf.common.initialization.batchnorm_adaptation import BatchnormAdaptationAlgorithm @@ -122,6 +123,7 @@ def get_types_of_grouping_ops(self) -> List[str]: return PTElementwisePruningOp.get_all_op_aliases() +@api() @ADAPTIVE_COMPRESSION_CONTROLLERS.register('pt_filter_pruning') class FilterPruningController(BasePruningAlgoController): def __init__(self, target_model: NNCFNetwork, diff --git a/nncf/torch/quantization/algo.py b/nncf/torch/quantization/algo.py index 90b1c80dc73..3bed0f70805 100644 --- a/nncf/torch/quantization/algo.py +++ b/nncf/torch/quantization/algo.py @@ -35,6 +35,7 @@ from nncf.api.compression import CompressionLoss from nncf.api.compression import CompressionScheduler from nncf.api.compression import CompressionStage +from nncf.common.api_marker import api from nncf.common.graph import NNCFGraph from nncf.common.graph import NNCFNode from nncf.common.graph import NNCFNodeName @@ -1252,6 +1253,7 @@ def init_range(self): raise NotImplementedError +@api() class QuantizationController(QuantizationControllerBase): def __init__(self, target_model: NNCFNetwork, config: NNCFConfig, diff --git a/nncf/torch/sparsity/base_algo.py b/nncf/torch/sparsity/base_algo.py index 12f45455b96..d668c237d06 100644 --- a/nncf/torch/sparsity/base_algo.py +++ b/nncf/torch/sparsity/base_algo.py @@ -17,6 +17,7 @@ from nncf.api.compression import CompressionLoss from nncf.api.compression import CompressionScheduler from nncf.api.compression import CompressionStage +from nncf.common.api_marker import api from nncf.common.graph import NNCFNode from nncf.common.graph import NNCFNodeName from nncf.common.graph.transformations.commands import TargetType @@ -93,6 +94,7 @@ def initialize(self, model: NNCFNetwork) -> None: pass +@api() class BaseSparsityAlgoController(PTCompressionAlgorithmController, SparsityController): def __init__(self, target_model: NNCFNetwork, sparsified_module_info: List[SparseModuleInfo]): super().__init__(target_model) diff --git a/nncf/torch/sparsity/const/algo.py b/nncf/torch/sparsity/const/algo.py index 46f28d725c3..020a3c2ac2d 100644 --- a/nncf/torch/sparsity/const/algo.py +++ b/nncf/torch/sparsity/const/algo.py @@ -12,6 +12,7 @@ """ from typing import Tuple +from nncf.common.api_marker import api from nncf.common.graph import NNCFNode from nncf.common.sparsity.statistics import ConstSparsityStatistics from nncf.common.statistics import NNCFStatistics @@ -35,6 +36,7 @@ def _are_frozen_layers_allowed(self) -> Tuple[bool, str]: return True, 'Frozen layers are allowed for const sparsity' +@api() class ConstSparsityController(BaseSparsityAlgoController): def freeze(self): pass diff --git a/nncf/torch/sparsity/magnitude/algo.py b/nncf/torch/sparsity/magnitude/algo.py index 01936432eaa..b88552e3695 100644 --- a/nncf/torch/sparsity/magnitude/algo.py +++ b/nncf/torch/sparsity/magnitude/algo.py @@ -18,6 +18,7 @@ from nncf import NNCFConfig from nncf.api.compression import CompressionStage from nncf.common.accuracy_aware_training.training_loop import ADAPTIVE_COMPRESSION_CONTROLLERS +from nncf.common.api_marker import api from nncf.common.graph import NNCFNode from nncf.common.initialization.batchnorm_adaptation import BatchnormAdaptationAlgorithm from nncf.common.schedulers import StubCompressionScheduler @@ -51,6 +52,7 @@ def _build_controller(self, model: NNCFNetwork) -> PTCompressionAlgorithmControl return MagnitudeSparsityController(model, self._sparsified_module_info, self.config) +@api() @ADAPTIVE_COMPRESSION_CONTROLLERS.register('pt_magnitude_sparsity') class MagnitudeSparsityController(BaseSparsityAlgoController): def __init__(self, target_model: NNCFNetwork, sparsified_module_info: List[SparseModuleInfo], diff --git a/nncf/torch/sparsity/rb/algo.py b/nncf/torch/sparsity/rb/algo.py index e9be7b76999..c7b21e46350 100644 --- a/nncf/torch/sparsity/rb/algo.py +++ b/nncf/torch/sparsity/rb/algo.py @@ -17,6 +17,7 @@ import torch.distributed as dist from nncf import NNCFConfig +from nncf.common.api_marker import api from nncf.config.extractors import extract_algo_specific_config from nncf.config.schemata.defaults import SPARSITY_INIT from nncf.config.schemata.defaults import SPARSITY_LEVEL_SETTING_MODE @@ -48,6 +49,7 @@ def _build_controller(self, model: NNCFNetwork) -> PTCompressionAlgorithmControl return RBSparsityController(model, self._sparsified_module_info, self.config) +@api() @ADAPTIVE_COMPRESSION_CONTROLLERS.register('pt_rb_sparsity') class RBSparsityController(BaseSparsityAlgoController): def __init__(self, target_model: NNCFNetwork, sparsified_module_info: List[SparseModuleInfo], diff --git a/nncf/torch/structures.py b/nncf/torch/structures.py index 616e1518c89..5cacf5d3d98 100644 --- a/nncf/torch/structures.py +++ b/nncf/torch/structures.py @@ -17,9 +17,11 @@ from torch.nn.modules.loss import _Loss from torch.utils.data import DataLoader +from nncf.common.api_marker import api from nncf.config.structures import NNCFExtraConfigStruct +@api() class QuantizationPrecisionInitArgs(NNCFExtraConfigStruct): """ Stores arguments for initialization of quantization's bitwidth. @@ -55,6 +57,7 @@ def get_id(cls) -> str: return "quantization_precision_init_args" +@api() class AutoQPrecisionInitArgs(NNCFExtraConfigStruct): """ :param data_loader: 'data_loader' - provides an iterable over the given dataset. Instance of @@ -79,6 +82,7 @@ def get_id(cls) -> str: return "autoq_precision_init_args" +@api() class LeGRInitArgs(NNCFExtraConfigStruct): """ Stores arguments for learning global ranking in pruning algorithm. @@ -112,6 +116,7 @@ def get_id(cls) -> str: return "legr_init_args" +@api() class DistributedCallbacksArgs(NNCFExtraConfigStruct): """ A pair of callbacks that is needed for distributed training of the model: wrapping model with wrapping_callback for @@ -132,6 +137,7 @@ def get_id(cls) -> str: return "distributed_callbacks_args" +@api() class ExecutionParameters: """ Parameters that are necessary for distributed training of the model. From 0065a278cbb9e3d89be66ec488610cb7ddf16135 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Wed, 5 Apr 2023 14:40:55 +0200 Subject: [PATCH 07/19] AutoAPI + FQN collection --- docs/api/collect_api_entities.py | 53 ------------ docs/api/source/_templates/class.rst | 34 ++++++++ docs/api/source/_templates/module.rst | 66 +++++++++++++++ docs/api/source/conf.py | 80 +++++++++++++++++-- .../source/{index_template.rst => index.rst} | 5 -- 5 files changed, 173 insertions(+), 65 deletions(-) delete mode 100644 docs/api/collect_api_entities.py create mode 100644 docs/api/source/_templates/class.rst create mode 100644 docs/api/source/_templates/module.rst rename docs/api/source/{index_template.rst => index.rst} (81%) diff --git a/docs/api/collect_api_entities.py b/docs/api/collect_api_entities.py deleted file mode 100644 index d98f3749f67..00000000000 --- a/docs/api/collect_api_entities.py +++ /dev/null @@ -1,53 +0,0 @@ -import importlib -import inspect -import pkgutil -from pathlib import Path - -import nncf - -modules = {} -skipped_modules = [] -for importer, modname, ispkg in pkgutil.walk_packages(path=nncf.__path__, - prefix=nncf.__name__+'.', - onerror=lambda x: None): - try: - modules[modname] = importlib.import_module(modname) - except: - skipped_modules.append(modname) - -api_fqns = [] -for modname, module in modules.items(): - print(f"{modname}") - for obj_name, obj in inspect.getmembers(module): - objects_module = getattr(obj, '__module__', None) - if objects_module == modname: - if inspect.isclass(obj) or inspect.isfunction(obj): - if hasattr(obj, "_nncf_api_marker"): - print(f"\t{obj_name}") - api_fqns.append(f"{modname}.{obj_name}") - -print() -skipped_str = '\n'.join(skipped_modules) -print(f"Skipped: {skipped_str}\n") - -print("API entities:") -for api_fqn in api_fqns: - print(api_fqn) - -DOC_ROOT = Path(__file__).parent -template_file = DOC_ROOT / 'source' / 'index_template.rst' -target_file = DOC_ROOT / 'source' / 'index.rst' - -with open(template_file, encoding='utf-8', mode='r') as f: - old_lines = f.readlines() - for idx, line in enumerate(old_lines): - anchor_line = idx - if line == '.. API_ENTITIES_TEMPLATE_ANCHOR' + '\n': - break - api_section = "" - for api_fqn in api_fqns: - api_section += f" {api_fqn}\n" - content = ''.join(old_lines[:anchor_line]) + api_section + ''.join(old_lines[anchor_line + 1:]) - -with open(target_file, encoding='utf-8', mode='w+') as f: - f.write(content) diff --git a/docs/api/source/_templates/class.rst b/docs/api/source/_templates/class.rst new file mode 100644 index 00000000000..8c5f475cac2 --- /dev/null +++ b/docs/api/source/_templates/class.rst @@ -0,0 +1,34 @@ +{{ fullname | escape | underline}} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :show-inheritance: + :inherited-members: + :special-members: __call__ + + {% block methods %} + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + :nosignatures: + {% for item in methods %} + {%- if not item.startswith('_') %} + ~{{ name }}.{{ item }} + {%- endif -%} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/docs/api/source/_templates/module.rst b/docs/api/source/_templates/module.rst new file mode 100644 index 00000000000..f63d8860d1d --- /dev/null +++ b/docs/api/source/_templates/module.rst @@ -0,0 +1,66 @@ +{{ fullname | escape | underline}} + +.. automodule:: {{ fullname }} + + {% block attributes %} + {% if attributes %} + .. rubric:: Module attributes + + .. autosummary:: + :toctree: + {% for item in attributes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + .. autosummary:: + :toctree: + :nosignatures: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: {{ _('Classes') }} + + .. autosummary:: + :toctree: + :template: class.rst + :nosignatures: + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + :toctree: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. autosummary:: + :toctree: + :template: custom-module-template.rst + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index e49f9aec27d..a28f6cf2e97 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -2,11 +2,17 @@ # # For the full list of built-in configuration values, see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +import importlib import inspect # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information import os +import pkgutil import sys +from typing import List + +import nncf + sys.path.insert(0, os.path.abspath('../../..')) project = 'nncf' @@ -18,20 +24,80 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = ['sphinx.ext.autodoc', - 'sphinx.ext.autosummary'] +extensions = ['autoapi.extension'] -autosummary_generate = True # Turn on sphinx.ext.autosummary +autoapi_dirs = ['../../../nncf'] +autoapi_options = ['members', 'show-inheritance', + 'show-module-summary', 'special-members', 'imported-members'] -templates_path = ['_templates'] +autoapi_template_dir = '_autoapi_templates' exclude_patterns = [] -module_fqn_with_api_fns_memo = set() -_memo = {} +def collect_api_entities() -> List[str]: + modules = {} + skipped_modules = [] + for importer, modname, ispkg in pkgutil.walk_packages(path=nncf.__path__, + prefix=nncf.__name__+'.', + onerror=lambda x: None): + try: + modules[modname] = importlib.import_module(modname) + except: + skipped_modules.append(modname) + + api_fqns = [] + for modname, module in modules.items(): + print(f"{modname}") + for obj_name, obj in inspect.getmembers(module): + objects_module = getattr(obj, '__module__', None) + if objects_module == modname: + if inspect.isclass(obj) or inspect.isfunction(obj): + if hasattr(obj, "_nncf_api_marker"): + print(f"\t{obj_name}") + api_fqns.append(f"{modname}.{obj_name}") + + print() + skipped_str = '\n'.join(skipped_modules) + print(f"Skipped: {skipped_str}\n") + + print("API entities:") + for api_fqn in api_fqns: + print(api_fqn) + return api_fqns + +api_fqns = collect_api_entities() + + +module_fqns = set() + +for fqn in api_fqns: + path_elements = fqn.split('.') + for i in range(1, len(path_elements)): + intermediate_module_path = '.'.join(path_elements[:i]) + module_fqns.add(intermediate_module_path) + + +def skip_non_api(app, what, name, obj, skip, options): + if what in ["module", "package"] and name in module_fqns: + print(f"VSHAMPOR: keeping module {name}") + return skip + if what in ["method", "attribute"]: + class_name = name.rpartition('.')[0] + if class_name in api_fqns: + return skip + if name not in api_fqns: + skip = True + else: + print(f"VSHAMPOR: keeping API entity {name}") + return skip + + +def setup(sphinx): + sphinx.connect("autoapi-skip-member", skip_non_api) + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'sphinx_book_theme' +html_theme = 'sphinx_rtd_theme' html_static_path = ['_static'] diff --git a/docs/api/source/index_template.rst b/docs/api/source/index.rst similarity index 81% rename from docs/api/source/index_template.rst rename to docs/api/source/index.rst index 01b6ddd89fe..cbfb6d6a5aa 100644 --- a/docs/api/source/index_template.rst +++ b/docs/api/source/index.rst @@ -10,8 +10,3 @@ Welcome to nncf's documentation! :maxdepth: 2 :caption: Contents: - -.. autosummary:: - :toctree: _autosummary - -.. API_ENTITIES_TEMPLATE_ANCHOR \ No newline at end of file From 7316a6c0a8bbcead0a8b6fee944fdeb2056ef702 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Tue, 11 Apr 2023 16:18:48 +0200 Subject: [PATCH 08/19] Switch to book theme and improve formatting --- docs/api/source/_autoapi_templates/index.rst | 13 ++ .../_autoapi_templates/python/attribute.rst | 1 + .../_autoapi_templates/python/class.rst | 58 +++++++++ .../source/_autoapi_templates/python/data.rst | 37 ++++++ .../_autoapi_templates/python/exception.rst | 1 + .../_autoapi_templates/python/function.rst | 15 +++ .../_autoapi_templates/python/method.rst | 19 +++ .../_autoapi_templates/python/module.rst | 112 ++++++++++++++++++ .../_autoapi_templates/python/package.rst | 1 + .../_autoapi_templates/python/property.rst | 15 +++ docs/api/source/_templates/class.rst | 34 ------ docs/api/source/_templates/module.rst | 66 ----------- docs/api/source/conf.py | 9 +- docs/api/source/index.rst | 6 +- nncf/torch/automl/__init__.py | 12 ++ nncf/torch/automl/agent/__init__.py | 12 ++ nncf/torch/automl/agent/ddpg/__init__.py | 12 ++ nncf/torch/automl/environment/__init__.py | 12 ++ nncf/torch/statistics/__init__.py | 12 ++ 19 files changed, 344 insertions(+), 103 deletions(-) create mode 100644 docs/api/source/_autoapi_templates/index.rst create mode 100644 docs/api/source/_autoapi_templates/python/attribute.rst create mode 100644 docs/api/source/_autoapi_templates/python/class.rst create mode 100644 docs/api/source/_autoapi_templates/python/data.rst create mode 100644 docs/api/source/_autoapi_templates/python/exception.rst create mode 100644 docs/api/source/_autoapi_templates/python/function.rst create mode 100644 docs/api/source/_autoapi_templates/python/method.rst create mode 100644 docs/api/source/_autoapi_templates/python/module.rst create mode 100644 docs/api/source/_autoapi_templates/python/package.rst create mode 100644 docs/api/source/_autoapi_templates/python/property.rst delete mode 100644 docs/api/source/_templates/class.rst delete mode 100644 docs/api/source/_templates/module.rst create mode 100644 nncf/torch/automl/__init__.py create mode 100644 nncf/torch/automl/agent/__init__.py create mode 100644 nncf/torch/automl/agent/ddpg/__init__.py create mode 100644 nncf/torch/automl/environment/__init__.py create mode 100644 nncf/torch/statistics/__init__.py diff --git a/docs/api/source/_autoapi_templates/index.rst b/docs/api/source/_autoapi_templates/index.rst new file mode 100644 index 00000000000..97687746ed0 --- /dev/null +++ b/docs/api/source/_autoapi_templates/index.rst @@ -0,0 +1,13 @@ +NNCF API Reference +============= + +.. toctree:: + :titlesonly: + + {% for page in pages %} + {% if page.top_level_object and page.display %} + {{ page.include_path }} + {% endif %} + {% endfor %} + + diff --git a/docs/api/source/_autoapi_templates/python/attribute.rst b/docs/api/source/_autoapi_templates/python/attribute.rst new file mode 100644 index 00000000000..ebaba555adc --- /dev/null +++ b/docs/api/source/_autoapi_templates/python/attribute.rst @@ -0,0 +1 @@ +{% extends "python/data.rst" %} diff --git a/docs/api/source/_autoapi_templates/python/class.rst b/docs/api/source/_autoapi_templates/python/class.rst new file mode 100644 index 00000000000..df5edffb62e --- /dev/null +++ b/docs/api/source/_autoapi_templates/python/class.rst @@ -0,0 +1,58 @@ +{% if obj.display %} +.. py:{{ obj.type }}:: {{ obj.short_name }}{% if obj.args %}({{ obj.args }}){% endif %} +{% for (args, return_annotation) in obj.overloads %} + {{ " " * (obj.type | length) }} {{ obj.short_name }}{% if args %}({{ args }}){% endif %} +{% endfor %} + + + {% if obj.bases %} + {% if "show-inheritance" in autoapi_options %} + Bases: {% for base in obj.bases %}{{ base|link_objs }}{% if not loop.last %}, {% endif %}{% endfor %} + {% endif %} + + + {% if "show-inheritance-diagram" in autoapi_options and obj.bases != ["object"] %} + .. autoapi-inheritance-diagram:: {{ obj.obj["full_name"] }} + :parts: 1 + {% if "private-members" in autoapi_options %} + :private-bases: + {% endif %} + + {% endif %} + {% endif %} + {% if obj.docstring %} + {{ obj.docstring|indent(3) }} + {% endif %} + {% if "inherited-members" in autoapi_options %} + {% set visible_classes = obj.classes|selectattr("display")|list %} + {% else %} + {% set visible_classes = obj.classes|rejectattr("inherited")|selectattr("display")|list %} + {% endif %} + {% for klass in visible_classes %} + {{ klass.render()|indent(3) }} + {% endfor %} + {% if "inherited-members" in autoapi_options %} + {% set visible_properties = obj.properties|selectattr("display")|list %} + {% else %} + {% set visible_properties = obj.properties|rejectattr("inherited")|selectattr("display")|list %} + {% endif %} + {% for property in visible_properties %} + {{ property.render()|indent(3) }} + {% endfor %} + {% if "inherited-members" in autoapi_options %} + {% set visible_attributes = obj.attributes|selectattr("display")|list %} + {% else %} + {% set visible_attributes = obj.attributes|rejectattr("inherited")|selectattr("display")|list %} + {% endif %} + {% for attribute in visible_attributes %} + {{ attribute.render()|indent(3) }} + {% endfor %} + {% if "inherited-members" in autoapi_options %} + {% set visible_methods = obj.methods|selectattr("display")|list %} + {% else %} + {% set visible_methods = obj.methods|rejectattr("inherited")|selectattr("display")|list %} + {% endif %} + {% for method in visible_methods %} + {{ method.render()|indent(3) }} + {% endfor %} +{% endif %} diff --git a/docs/api/source/_autoapi_templates/python/data.rst b/docs/api/source/_autoapi_templates/python/data.rst new file mode 100644 index 00000000000..3d12b2d0c7c --- /dev/null +++ b/docs/api/source/_autoapi_templates/python/data.rst @@ -0,0 +1,37 @@ +{% if obj.display %} +.. py:{{ obj.type }}:: {{ obj.name }} + {%- if obj.annotation is not none %} + + :type: {%- if obj.annotation %} {{ obj.annotation }}{%- endif %} + + {%- endif %} + + {%- if obj.value is not none %} + + :value: {% if obj.value is string and obj.value.splitlines()|count > 1 -%} + Multiline-String + + .. raw:: html + +
Show Value + + .. code-block:: python + + """{{ obj.value|indent(width=8,blank=true) }}""" + + .. raw:: html + +
+ + {%- else -%} + {%- if obj.value is string -%} + {{ "%r" % obj.value|string|truncate(100) }} + {%- else -%} + {{ obj.value|string|truncate(100) }} + {%- endif -%} + {%- endif %} + {%- endif %} + + + {{ obj.docstring|indent(3) }} +{% endif %} diff --git a/docs/api/source/_autoapi_templates/python/exception.rst b/docs/api/source/_autoapi_templates/python/exception.rst new file mode 100644 index 00000000000..92f3d38fd5f --- /dev/null +++ b/docs/api/source/_autoapi_templates/python/exception.rst @@ -0,0 +1 @@ +{% extends "python/class.rst" %} diff --git a/docs/api/source/_autoapi_templates/python/function.rst b/docs/api/source/_autoapi_templates/python/function.rst new file mode 100644 index 00000000000..b00d5c24454 --- /dev/null +++ b/docs/api/source/_autoapi_templates/python/function.rst @@ -0,0 +1,15 @@ +{% if obj.display %} +.. py:function:: {{ obj.short_name }}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %} + +{% for (args, return_annotation) in obj.overloads %} + {{ obj.short_name }}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %} + +{% endfor %} + {% for property in obj.properties %} + :{{ property }}: + {% endfor %} + + {% if obj.docstring %} + {{ obj.docstring|indent(3) }} + {% endif %} +{% endif %} diff --git a/docs/api/source/_autoapi_templates/python/method.rst b/docs/api/source/_autoapi_templates/python/method.rst new file mode 100644 index 00000000000..723cb7bbe54 --- /dev/null +++ b/docs/api/source/_autoapi_templates/python/method.rst @@ -0,0 +1,19 @@ +{%- if obj.display %} +.. py:method:: {{ obj.short_name }}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %} + +{% for (args, return_annotation) in obj.overloads %} + {{ obj.short_name }}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %} + +{% endfor %} + {% if obj.properties %} + {% for property in obj.properties %} + :{{ property }}: + {% endfor %} + + {% else %} + + {% endif %} + {% if obj.docstring %} + {{ obj.docstring|indent(3) }} + {% endif %} +{% endif %} diff --git a/docs/api/source/_autoapi_templates/python/module.rst b/docs/api/source/_autoapi_templates/python/module.rst new file mode 100644 index 00000000000..a95cfb6c0e3 --- /dev/null +++ b/docs/api/source/_autoapi_templates/python/module.rst @@ -0,0 +1,112 @@ +{% if not obj.display %} +:orphan: + +{% endif %} +:py:mod:`{{ obj.name }}` +=========={{ "=" * obj.name|length }} + +.. py:module:: {{ obj.name }} + +{% if obj.docstring %} +.. autoapi-nested-parse:: + + {{ obj.docstring|indent(3) }} + +{% endif %} + +{% block subpackages %} +{% set visible_subpackages = obj.subpackages|selectattr("display")|list %} +{% if visible_subpackages %} +Subpackages +----------- +.. toctree:: + :titlesonly: + :maxdepth: 3 + +{% for subpackage in visible_subpackages %} + {{ subpackage.short_name }}/index.rst +{% endfor %} + + +{% endif %} +{% endblock %} +{% block submodules %} +{% set visible_submodules = obj.submodules|selectattr("display")|list %} +{% if visible_submodules %} +Submodules +---------- +.. toctree:: + :titlesonly: + :maxdepth: 1 + +{% for submodule in visible_submodules %} + {{ submodule.short_name }}/index.rst +{% endfor %} + + +{% endif %} +{% endblock %} +{% block content %} +{% if obj.all is not none %} +{% set visible_children = obj.children|selectattr("short_name", "in", obj.all)|list %} +{% elif obj.type is equalto("package") %} +{% set visible_children = obj.children|selectattr("display")|list %} +{% else %} +{% set visible_children = obj.children|selectattr("display")|rejectattr("imported")|list %} +{% endif %} +{% if visible_children %} + +{% set visible_classes = visible_children|selectattr("type", "equalto", "class")|list %} +{% set visible_functions = visible_children|selectattr("type", "equalto", "function")|list %} +{% set visible_attributes = visible_children|selectattr("type", "equalto", "data")|list %} +{% if "show-module-summary" in autoapi_options and (visible_classes or visible_functions) %} +{% block classes scoped %} +{% if visible_classes %} +Classes +~~~~~~~ + +.. autoapisummary:: + +{% for klass in visible_classes %} + {{ klass.id }} +{% endfor %} + + +{% endif %} +{% endblock %} + +{% block functions scoped %} +{% if visible_functions %} +Functions +~~~~~~~~~ + +.. autoapisummary:: + +{% for function in visible_functions %} + {{ function.id }} +{% endfor %} + + +{% endif %} +{% endblock %} + +{% block attributes scoped %} +{% if visible_attributes %} +Attributes +~~~~~~~~~~ + +.. autoapisummary:: + +{% for attribute in visible_attributes %} + {{ attribute.id }} +{% endfor %} + + +{% endif %} +{% endblock %} +{% endif %} +{% for obj_item in visible_children %} +{{ obj_item.render()|indent(0) }} +{% endfor %} +{% endif %} +{% endblock %} diff --git a/docs/api/source/_autoapi_templates/python/package.rst b/docs/api/source/_autoapi_templates/python/package.rst new file mode 100644 index 00000000000..fb9a64965e9 --- /dev/null +++ b/docs/api/source/_autoapi_templates/python/package.rst @@ -0,0 +1 @@ +{% extends "python/module.rst" %} diff --git a/docs/api/source/_autoapi_templates/python/property.rst b/docs/api/source/_autoapi_templates/python/property.rst new file mode 100644 index 00000000000..70af24236f1 --- /dev/null +++ b/docs/api/source/_autoapi_templates/python/property.rst @@ -0,0 +1,15 @@ +{%- if obj.display %} +.. py:property:: {{ obj.short_name }} + {% if obj.annotation %} + :type: {{ obj.annotation }} + {% endif %} + {% if obj.properties %} + {% for property in obj.properties %} + :{{ property }}: + {% endfor %} + {% endif %} + + {% if obj.docstring %} + {{ obj.docstring|indent(3) }} + {% endif %} +{% endif %} diff --git a/docs/api/source/_templates/class.rst b/docs/api/source/_templates/class.rst deleted file mode 100644 index 8c5f475cac2..00000000000 --- a/docs/api/source/_templates/class.rst +++ /dev/null @@ -1,34 +0,0 @@ -{{ fullname | escape | underline}} - -.. currentmodule:: {{ module }} - -.. autoclass:: {{ objname }} - :members: - :show-inheritance: - :inherited-members: - :special-members: __call__ - - {% block methods %} - {% if methods %} - .. rubric:: {{ _('Methods') }} - - .. autosummary:: - :nosignatures: - {% for item in methods %} - {%- if not item.startswith('_') %} - ~{{ name }}.{{ item }} - {%- endif -%} - {%- endfor %} - {% endif %} - {% endblock %} - - {% block attributes %} - {% if attributes %} - .. rubric:: {{ _('Attributes') }} - - .. autosummary:: - {% for item in attributes %} - ~{{ name }}.{{ item }} - {%- endfor %} - {% endif %} - {% endblock %} diff --git a/docs/api/source/_templates/module.rst b/docs/api/source/_templates/module.rst deleted file mode 100644 index f63d8860d1d..00000000000 --- a/docs/api/source/_templates/module.rst +++ /dev/null @@ -1,66 +0,0 @@ -{{ fullname | escape | underline}} - -.. automodule:: {{ fullname }} - - {% block attributes %} - {% if attributes %} - .. rubric:: Module attributes - - .. autosummary:: - :toctree: - {% for item in attributes %} - {{ item }} - {%- endfor %} - {% endif %} - {% endblock %} - - {% block functions %} - {% if functions %} - .. rubric:: {{ _('Functions') }} - - .. autosummary:: - :toctree: - :nosignatures: - {% for item in functions %} - {{ item }} - {%- endfor %} - {% endif %} - {% endblock %} - - {% block classes %} - {% if classes %} - .. rubric:: {{ _('Classes') }} - - .. autosummary:: - :toctree: - :template: class.rst - :nosignatures: - {% for item in classes %} - {{ item }} - {%- endfor %} - {% endif %} - {% endblock %} - - {% block exceptions %} - {% if exceptions %} - .. rubric:: {{ _('Exceptions') }} - - .. autosummary:: - :toctree: - {% for item in exceptions %} - {{ item }} - {%- endfor %} - {% endif %} - {% endblock %} - -{% block modules %} -{% if modules %} -.. autosummary:: - :toctree: - :template: custom-module-template.rst - :recursive: -{% for item in modules %} - {{ item }} -{%- endfor %} -{% endif %} -{% endblock %} diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index a28f6cf2e97..ad1de033175 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -31,6 +31,13 @@ 'show-module-summary', 'special-members', 'imported-members'] autoapi_template_dir = '_autoapi_templates' +autoapi_keep_files = True +autoapi_add_toctree_entry = False + +html_theme_options = { + 'navigation_depth': -1, +} + exclude_patterns = [] @@ -99,5 +106,5 @@ def setup(sphinx): # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -html_theme = 'sphinx_rtd_theme' +html_theme = 'sphinx_book_theme' html_static_path = ['_static'] diff --git a/docs/api/source/index.rst b/docs/api/source/index.rst index cbfb6d6a5aa..cdfcaf88037 100644 --- a/docs/api/source/index.rst +++ b/docs/api/source/index.rst @@ -3,10 +3,12 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to nncf's documentation! +NNCF doc testing! ================================ .. toctree:: - :maxdepth: 2 :caption: Contents: + :maxdepth: 3 + + API Reference diff --git a/nncf/torch/automl/__init__.py b/nncf/torch/automl/__init__.py new file mode 100644 index 00000000000..de0da022181 --- /dev/null +++ b/nncf/torch/automl/__init__.py @@ -0,0 +1,12 @@ +""" + Copyright (c) 2023 Intel Corporation + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" \ No newline at end of file diff --git a/nncf/torch/automl/agent/__init__.py b/nncf/torch/automl/agent/__init__.py new file mode 100644 index 00000000000..de0da022181 --- /dev/null +++ b/nncf/torch/automl/agent/__init__.py @@ -0,0 +1,12 @@ +""" + Copyright (c) 2023 Intel Corporation + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" \ No newline at end of file diff --git a/nncf/torch/automl/agent/ddpg/__init__.py b/nncf/torch/automl/agent/ddpg/__init__.py new file mode 100644 index 00000000000..de0da022181 --- /dev/null +++ b/nncf/torch/automl/agent/ddpg/__init__.py @@ -0,0 +1,12 @@ +""" + Copyright (c) 2023 Intel Corporation + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" \ No newline at end of file diff --git a/nncf/torch/automl/environment/__init__.py b/nncf/torch/automl/environment/__init__.py new file mode 100644 index 00000000000..de0da022181 --- /dev/null +++ b/nncf/torch/automl/environment/__init__.py @@ -0,0 +1,12 @@ +""" + Copyright (c) 2023 Intel Corporation + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" \ No newline at end of file diff --git a/nncf/torch/statistics/__init__.py b/nncf/torch/statistics/__init__.py new file mode 100644 index 00000000000..de0da022181 --- /dev/null +++ b/nncf/torch/statistics/__init__.py @@ -0,0 +1,12 @@ +""" + Copyright (c) 2023 Intel Corporation + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" \ No newline at end of file From 01bc904eab851df68fae3ac805ce333fa978150e Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Tue, 11 Apr 2023 17:10:14 +0200 Subject: [PATCH 09/19] Mock TF imports --- docs/api/requirements.txt | 3 +++ docs/api/source/conf.py | 6 +++++- nncf/onnx/engine.py | 3 ++- nncf/openvino/engine.py | 2 ++ nncf/tensorflow/__init__.py | 11 ++++++++--- 5 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 docs/api/requirements.txt diff --git a/docs/api/requirements.txt b/docs/api/requirements.txt new file mode 100644 index 00000000000..b1f0c1a53ec --- /dev/null +++ b/docs/api/requirements.txt @@ -0,0 +1,3 @@ +Sphinx==6.1.3 +sphinx-autoapi==2.1.0 +sphinx-book-theme==1.0.1 \ No newline at end of file diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index ad1de033175..9a10cbe2125 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -11,6 +11,8 @@ import sys from typing import List +from sphinx.ext.autodoc import mock + import nncf sys.path.insert(0, os.path.abspath('../../..')) @@ -72,7 +74,9 @@ def collect_api_entities() -> List[str]: print(api_fqn) return api_fqns -api_fqns = collect_api_entities() + +with mock(['tensorflow', 'tensorflow_addons']): + api_fqns = collect_api_entities() module_fqns = set() diff --git a/nncf/onnx/engine.py b/nncf/onnx/engine.py index 555783a6bc7..02549f043f6 100644 --- a/nncf/onnx/engine.py +++ b/nncf/onnx/engine.py @@ -16,9 +16,10 @@ import numpy as np import onnxruntime as rt +from nncf.common.api_marker import api from nncf.common.engine import Engine - +@api() class ONNXEngine(Engine): """ Engine for ONNX backend using ONNXRuntime to infer the model. diff --git a/nncf/openvino/engine.py b/nncf/openvino/engine.py index ec94a18c123..3e03ec8b78e 100644 --- a/nncf/openvino/engine.py +++ b/nncf/openvino/engine.py @@ -22,6 +22,7 @@ import openvino.runtime as ov from nncf.api.compression import TModel +from nncf.common.api_marker import api from nncf.data import Dataset from nncf.data.dataset import DataProvider @@ -116,6 +117,7 @@ def calc_per_sample_metrics(compiled_model: ov.CompiledModel, # TODO(andrey-churkin): This class should be refactored. We will be able to do that when # we will have POT code in NNCF. +@api() class OVEngine(pot.IEEngine): """ Implementation of the engine for OpenVINO backend. diff --git a/nncf/tensorflow/__init__.py b/nncf/tensorflow/__init__.py index 5f98c807144..bd3eb2bb55a 100644 --- a/nncf/tensorflow/__init__.py +++ b/nncf/tensorflow/__init__.py @@ -18,14 +18,19 @@ import tensorflow from pkg_resources import parse_version -tensorflow_version = parse_version(tensorflow.__version__).base_version +try: + tensorflow_version = parse_version(_tf_version).base_version +except: + nncf_logger.debug("Could not parse tensorflow version") + _tf_version = '0.0.0' + tensorflow_version = parse_version(_tf_version).base_version tensorflow_version_major, tensorflow_version_minor = tuple(map(int, tensorflow_version.split('.')))[:2] if not tensorflow_version.startswith(BKC_TF_VERSION[:-2]): - warn_bkc_version_mismatch("tensorflow", BKC_TF_VERSION, tensorflow.__version__) + warn_bkc_version_mismatch("tensorflow", BKC_TF_VERSION, _tf_version) elif not (tensorflow_version_major == 2 and 4 <= tensorflow_version_minor <= 11): raise RuntimeError( f'NNCF only supports 2.4.0 <= tensorflow <= 2.11.*, ' - f'while current tensorflow version is {tensorflow.__version__}') + f'while current tensorflow version is {_tf_version}') from nncf.tensorflow.helpers import create_compressed_model From 4b21ec587570ab4e1f87d8769762becb948b174e Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Tue, 11 Apr 2023 17:17:48 +0200 Subject: [PATCH 10/19] Comments --- docs/api/source/conf.py | 27 +++++++++++---------------- docs/api/source/index.rst | 6 +----- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index 9a10cbe2125..d8a2f8bb9d9 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -1,11 +1,5 @@ -# Configuration file for the Sphinx documentation builder. -# -# For the full list of built-in configuration values, see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html import importlib import inspect -# -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information import os import pkgutil import sys @@ -22,9 +16,6 @@ author = 'Intel Corporation' release = 'v2.4.0' -# -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration - extensions = ['autoapi.extension'] @@ -44,6 +35,11 @@ def collect_api_entities() -> List[str]: + """ + Collects the fully qualified names of symbols in NNCF package that contain a special attribute (set via + `nncf.common.api_marker.api` decorator) marking them as API entities. + :return: A list of fully qualified names of API symbols. + """ modules = {} skipped_modules = [] for importer, modname, ispkg in pkgutil.walk_packages(path=nncf.__path__, @@ -75,10 +71,9 @@ def collect_api_entities() -> List[str]: return api_fqns -with mock(['tensorflow', 'tensorflow_addons']): +with mock(['torch', 'onnx', 'openvino', 'tensorflow', 'tensorflow_addons']): api_fqns = collect_api_entities() - module_fqns = set() for fqn in api_fqns: @@ -89,8 +84,11 @@ def collect_api_entities() -> List[str]: def skip_non_api(app, what, name, obj, skip, options): + # AutoAPI-allowed callback to skip certain elements from generated documentation. + # We use it to only allow API entities in the documentation (otherwise AutoAPI would generate docs for every + # non-private symbol available in NNCF) if what in ["module", "package"] and name in module_fqns: - print(f"VSHAMPOR: keeping module {name}") + print(f"skip_non_api: keeping module {name}") return skip if what in ["method", "attribute"]: class_name = name.rpartition('.')[0] @@ -99,7 +97,7 @@ def skip_non_api(app, what, name, obj, skip, options): if name not in api_fqns: skip = True else: - print(f"VSHAMPOR: keeping API entity {name}") + print(f"skip_non_api: keeping API entity {name}") return skip @@ -107,8 +105,5 @@ def setup(sphinx): sphinx.connect("autoapi-skip-member", skip_non_api) -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output - html_theme = 'sphinx_book_theme' html_static_path = ['_static'] diff --git a/docs/api/source/index.rst b/docs/api/source/index.rst index cdfcaf88037..37951ac42d5 100644 --- a/docs/api/source/index.rst +++ b/docs/api/source/index.rst @@ -1,8 +1,3 @@ -.. nncf documentation master file, created by - sphinx-quickstart on Fri Mar 24 11:30:33 2023. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - NNCF doc testing! ================================ @@ -11,4 +6,5 @@ NNCF doc testing! :maxdepth: 3 API Reference + JSON configuration file schema<./schema.html#http://> From 54dfe6b8d209d1faa3b703aca1f82c498592318b Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Tue, 11 Apr 2023 17:35:46 +0200 Subject: [PATCH 11/19] Add GH workflows for doc builds and API checks --- .github/{ => action_configs}/labeler.yml | 0 .github/workflows/api_changes_check.yml | 42 +++++++++++++++++++++ .github/workflows/build_and_publish_doc.yml | 41 ++++++++++++++++++++ .github/workflows/build_html_doc.yml | 21 +++++++++++ .github/workflows/build_schema_page.yml | 25 ++++++------ .github/workflows/labeler.yml | 4 +- .github/workflows/python-publish.yml | 2 +- docs/api/source/index.rst | 4 +- nncf/onnx/engine.py | 2 - nncf/openvino/engine.py | 2 - 10 files changed, 120 insertions(+), 23 deletions(-) rename .github/{ => action_configs}/labeler.yml (100%) create mode 100644 .github/workflows/api_changes_check.yml create mode 100644 .github/workflows/build_and_publish_doc.yml create mode 100644 .github/workflows/build_html_doc.yml diff --git a/.github/labeler.yml b/.github/action_configs/labeler.yml similarity index 100% rename from .github/labeler.yml rename to .github/action_configs/labeler.yml diff --git a/.github/workflows/api_changes_check.yml b/.github/workflows/api_changes_check.yml new file mode 100644 index 00000000000..b359dd06aab --- /dev/null +++ b/.github/workflows/api_changes_check.yml @@ -0,0 +1,42 @@ +name: API changes check +on: + pull_request_target: + branches: + - develop + +env: + API_DOC_HTML_ROOT_RELATIVE_PATH: autoapi + +concurrency: + group: ci-${{ github.head_ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + call-build-api-doc: + uses: ./.github/workflows/build_html_doc.yml + compare-api-doc-with-develop: + needs: call-build-api-doc + runs-on: ubuntu-latest + steps: + - name: Download built HTML doc as artifact from previous step + uses: alehechka/download-tartifact@v2 + with: + name: html_doc_artifact + - name: Checkout latest doc_pages branch tip + uses: actions/checkout@v3 + with: + ref: doc_pages + path: previous_doc_state + - name: Get changed file names in API doc path + id: diff + run: | + cd html_build/html/$API_DOC_HTML_ROOT_RELATIVE_PATH + find . -name "*.html" -exec diff -qrBZ {} $GITHUB_WORKSPACE/previous_doc_state/$API_DOC_HTML_ROOT_RELATIVE_PATH/{} \; + - uses: actions-ecosystem/action-add-labels@v1 + if: ${{ steps.diff.outputs }} != "" + with: + labels: API + - uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ steps.diff.outputs }} == "" + with: + labels: API diff --git a/.github/workflows/build_and_publish_doc.yml b/.github/workflows/build_and_publish_doc.yml new file mode 100644 index 00000000000..7500a1b604c --- /dev/null +++ b/.github/workflows/build_and_publish_doc.yml @@ -0,0 +1,41 @@ +name: Build complete docs and publish to GH Pages +on: + push: + branches: + - develop + +env: + GH_PAGES_BRANCH: doc_pages + +concurrency: + group: ci-${{ github.head_ref }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: write +jobs: + call-build-html-doc: + uses: ./.github/workflows/build_html_doc.yml + call-build-schema-page: + uses: ./.github/workflows/build_schema_page.yml + publish: + needs: [call-build-html-doc, call-build-schema-page] + runs-on: ubuntu-latest + steps: + - name: Checkout main repo # the github-pages-deploy-action seems to require this step + uses: actions/checkout@v3 + - name: Download HTML doc build artifact + uses: alehechka/download-tartifact@v2 + with: + name: html_doc_artifact + - name: Download schema doc build artifact + uses: alehechka/download-tartifact@v2 + with: + name: schema_doc_artifact + path: html_build/html + - name: Publish built docs on Github Pages branch ${{ env.GH_PAGES_BRANCH }} + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: html_build/html + token: ${{ secrets.PUSH_TO_GH_PAGES_BRANCH }} + branch: ${{ env.GH_PAGES_BRANCH }} diff --git a/.github/workflows/build_html_doc.yml b/.github/workflows/build_html_doc.yml new file mode 100644 index 00000000000..d07119ec631 --- /dev/null +++ b/.github/workflows/build_html_doc.yml @@ -0,0 +1,21 @@ +name: HTML documentation build +on: + workflow_call: +jobs: + build-html: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install NNCF and doc requirements + run: | + pip install -e . + pip install -r docs/api/requirements.txt + - name: Build API docs + run: | + sphinx-build -M html docs/api/source html_build + - name: Upload built HTMLs as job artifact + uses: alehechka/upload-tartifact@v2 + with: + name: html_doc_artifact + path: html_build/html diff --git a/.github/workflows/build_schema_page.yml b/.github/workflows/build_schema_page.yml index 395a6ea98c9..b1368792941 100644 --- a/.github/workflows/build_schema_page.yml +++ b/.github/workflows/build_schema_page.yml @@ -1,31 +1,28 @@ name: Config schema HTML build on: - push: + pull_request_target: branches: - develop - - test_for_doc_build_trigger paths: - nncf/config/** -permissions: - contents: write + workflow_call: jobs: - build-and-deploy: - concurrency: ci-${{ github.ref }} # Recommended if you intend to make multiple deployments in quick succession. + build-config-schema-html: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - - name: Install and Build # This example project is built using npm and outputs the result to the 'build' folder. Replace with the commands required to build your project, or remove this step entirely if your site is pre-built. + - name: Install and Build run: | pip install json-schema-for-humans pip install -e . python -c 'import jstyleson; from nncf.config import NNCFConfig; jstyleson.dump(NNCFConfig.schema(), open("./schema.json", "w"), indent=2)' - mkdir schema_html_build - generate-schema-doc --deprecated-from-description schema.json schema_html_build/index.html - - name: Deploy - uses: JamesIves/github-pages-deploy-action@v4 + mkdir schema + generate-schema-doc --deprecated-from-description schema.json schema/index.html + + - name: Upload result as artifact + uses: alehechka/upload-tartifact@v2 with: - folder: schema_html_build # The folder the action should deploy. - token: ${{ secrets.PUSH_TO_GH_PAGES_BRANCH }} - branch: doc_pages + name: schema_doc_artifact + path: schema diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 1b64d7304d2..c3eb8808fbd 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -2,7 +2,7 @@ name: "Pull Request Labeler" on: [pull_request_target] jobs: - triage: + set-label: permissions: contents: read pull-requests: write @@ -11,5 +11,5 @@ jobs: - uses: actions/labeler@v4 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" - configuration-path: '.github/labeler.yml' + configuration-path: '.github/action_configs/labeler.yml' sync-labels: true diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index b9882c135c1..94053830522 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -13,7 +13,7 @@ # This workflow will upload a Python Package using Twine when a release is created # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries -name: Upload Python Package +name: Publish release Python package to PyPI on: release: diff --git a/docs/api/source/index.rst b/docs/api/source/index.rst index 37951ac42d5..4d5159f8282 100644 --- a/docs/api/source/index.rst +++ b/docs/api/source/index.rst @@ -1,4 +1,4 @@ -NNCF doc testing! +NNCF documentation ================================ .. toctree:: @@ -6,5 +6,5 @@ NNCF doc testing! :maxdepth: 3 API Reference - JSON configuration file schema<./schema.html#http://> + JSON configuration file schema<./schema#http://> diff --git a/nncf/onnx/engine.py b/nncf/onnx/engine.py index 02549f043f6..9e76d8bc3fa 100644 --- a/nncf/onnx/engine.py +++ b/nncf/onnx/engine.py @@ -16,10 +16,8 @@ import numpy as np import onnxruntime as rt -from nncf.common.api_marker import api from nncf.common.engine import Engine -@api() class ONNXEngine(Engine): """ Engine for ONNX backend using ONNXRuntime to infer the model. diff --git a/nncf/openvino/engine.py b/nncf/openvino/engine.py index 3e03ec8b78e..ec94a18c123 100644 --- a/nncf/openvino/engine.py +++ b/nncf/openvino/engine.py @@ -22,7 +22,6 @@ import openvino.runtime as ov from nncf.api.compression import TModel -from nncf.common.api_marker import api from nncf.data import Dataset from nncf.data.dataset import DataProvider @@ -117,7 +116,6 @@ def calc_per_sample_metrics(compiled_model: ov.CompiledModel, # TODO(andrey-churkin): This class should be refactored. We will be able to do that when # we will have POT code in NNCF. -@api() class OVEngine(pot.IEEngine): """ Implementation of the engine for OpenVINO backend. From 95e8d8c56065dbd18a9d75653aab31c4c5da21fa Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Thu, 13 Apr 2023 18:10:17 +0200 Subject: [PATCH 12/19] Pylint --- nncf/torch/automl/__init__.py | 2 +- nncf/torch/automl/agent/__init__.py | 2 +- nncf/torch/automl/agent/ddpg/__init__.py | 2 +- nncf/torch/model_creation.py | 18 ++++++++++-------- nncf/torch/statistics/__init__.py | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/nncf/torch/automl/__init__.py b/nncf/torch/automl/__init__.py index de0da022181..8727b935935 100644 --- a/nncf/torch/automl/__init__.py +++ b/nncf/torch/automl/__init__.py @@ -9,4 +9,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -""" \ No newline at end of file +""" diff --git a/nncf/torch/automl/agent/__init__.py b/nncf/torch/automl/agent/__init__.py index de0da022181..8727b935935 100644 --- a/nncf/torch/automl/agent/__init__.py +++ b/nncf/torch/automl/agent/__init__.py @@ -9,4 +9,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -""" \ No newline at end of file +""" diff --git a/nncf/torch/automl/agent/ddpg/__init__.py b/nncf/torch/automl/agent/ddpg/__init__.py index de0da022181..8727b935935 100644 --- a/nncf/torch/automl/agent/ddpg/__init__.py +++ b/nncf/torch/automl/agent/ddpg/__init__.py @@ -9,4 +9,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -""" \ No newline at end of file +""" diff --git a/nncf/torch/model_creation.py b/nncf/torch/model_creation.py index cff847499e8..275de53dc65 100644 --- a/nncf/torch/model_creation.py +++ b/nncf/torch/model_creation.py @@ -151,20 +151,22 @@ def create_nncf_network(model: torch.nn.Module, the internal graph representation via tracing. Specifying this is useful when the original training pipeline has special formats of data loader output or has additional *forward* arguments other than input tensors. Otherwise, the *forward* call of the model during graph tracing will be made with mock tensors according - to the shape specified in the config object. The dummy_forward_fn code MUST contain calls to nncf.nncf_model_input + to the shape specified in the config object. The dummy_forward_fn code MUST contain calls to + nncf.nncf_model_input functions made with each compressed model input tensor in the underlying model's args/kwargs tuple, and these calls should be exactly the same as in the wrap_inputs_fn function code (see below); if dummy_forward_fn is specified, then wrap_inputs_fn also must be specified. :param wrap_inputs_fn: if supplied, will be used on the module's input arguments during a regular, non-dummy forward call before passing the inputs to the underlying compressed model. This is required if the model's input - tensors that are important for compression are not supplied as arguments to the model's forward call directly, but - instead are located in a container (such as list), and the model receives the container as an argument. + tensors that are important for compression are not supplied as arguments to the model's forward call directly, + but instead are located in a container (such as list), and the model receives the container as an argument. wrap_inputs_fn should take as input two arguments - the tuple of positional arguments to the underlying - model's forward call, and a dict of keyword arguments to the same. The function should wrap each tensor among the - supplied model's args and kwargs that is important for compression (e.g. quantization) with an nncf.nncf_model_input - function, which is a no-operation function and marks the tensors as inputs to be traced by NNCF in the internal - graph representation. Output is the tuple of (args, kwargs), where args and kwargs are the same as were supplied in - input, but each tensor in the original input. Must be specified if dummy_forward_fn is specified. + model's forward call, and a dict of keyword arguments to the same. The function should wrap each tensor among + the supplied model's args and kwargs that is important for compression (e.g. quantization) with an + nncf.nncf_model_input function, which is a no-operation function and marks the tensors as inputs to be traced + by NNCF in the internal graph representation. Output is the tuple of (args, kwargs), where args and kwargs are + the same as were supplied in input, but each tensor in the original input. Must be specified if + dummy_forward_fn is specified. :param wrap_outputs_fn: if supplied, will be used on the module's output during a regular, non-dummy forward call. :return: A model wrapped by NNCFNetwork, which is ready for adding compression. """ diff --git a/nncf/torch/statistics/__init__.py b/nncf/torch/statistics/__init__.py index de0da022181..8727b935935 100644 --- a/nncf/torch/statistics/__init__.py +++ b/nncf/torch/statistics/__init__.py @@ -9,4 +9,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -""" \ No newline at end of file +""" From c7a7e3d2421149eb84fa452b328c15d0d237671d Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Thu, 13 Apr 2023 18:22:55 +0200 Subject: [PATCH 13/19] Fix diff files check --- .github/workflows/api_changes_check.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/api_changes_check.yml b/.github/workflows/api_changes_check.yml index b359dd06aab..da803ac48f4 100644 --- a/.github/workflows/api_changes_check.yml +++ b/.github/workflows/api_changes_check.yml @@ -31,12 +31,14 @@ jobs: id: diff run: | cd html_build/html/$API_DOC_HTML_ROOT_RELATIVE_PATH - find . -name "*.html" -exec diff -qrBZ {} $GITHUB_WORKSPACE/previous_doc_state/$API_DOC_HTML_ROOT_RELATIVE_PATH/{} \; + CHANGED_FILES=$(find . -name "*.html" -exec diff -qrBZ {} $GITHUB_WORKSPACE/previous_doc_state/$API_DOC_HTML_ROOT_RELATIVE_PATH/{} \;) + echo ${CHANGED_FILES} + echo "changed_files=${CHANGED_FILES}" >> $GITHUB_OUTPUT - uses: actions-ecosystem/action-add-labels@v1 - if: ${{ steps.diff.outputs }} != "" + if: ${{ steps.diff.outputs.changed_files }} != "" with: labels: API - uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ steps.diff.outputs }} == "" + if: ${{ steps.diff.outputs.changed_files }} == "" with: labels: API From 73259899c1f03d454907298058b3d1831957a311 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Fri, 14 Apr 2023 10:50:29 +0200 Subject: [PATCH 14/19] Fix torch version parsing --- .github/workflows/api_changes_check.yml | 1 + docs/api/source/conf.py | 12 ++++++------ nncf/torch/__init__.py | 10 +++++++++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/api_changes_check.yml b/.github/workflows/api_changes_check.yml index da803ac48f4..08e457fdc6d 100644 --- a/.github/workflows/api_changes_check.yml +++ b/.github/workflows/api_changes_check.yml @@ -33,6 +33,7 @@ jobs: cd html_build/html/$API_DOC_HTML_ROOT_RELATIVE_PATH CHANGED_FILES=$(find . -name "*.html" -exec diff -qrBZ {} $GITHUB_WORKSPACE/previous_doc_state/$API_DOC_HTML_ROOT_RELATIVE_PATH/{} \;) echo ${CHANGED_FILES} + CHANGED_FILES=$(echo $CHANGED_FILES | tr '\n' ' ') echo "changed_files=${CHANGED_FILES}" >> $GITHUB_OUTPUT - uses: actions-ecosystem/action-add-labels@v1 if: ${{ steps.diff.outputs.changed_files }} != "" diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index d8a2f8bb9d9..d353c2da529 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -7,7 +7,6 @@ from sphinx.ext.autodoc import mock -import nncf sys.path.insert(0, os.path.abspath('../../..')) @@ -41,14 +40,15 @@ def collect_api_entities() -> List[str]: :return: A list of fully qualified names of API symbols. """ modules = {} - skipped_modules = [] + skipped_modules = {} # type: Dict[str, str] + import nncf for importer, modname, ispkg in pkgutil.walk_packages(path=nncf.__path__, prefix=nncf.__name__+'.', onerror=lambda x: None): try: modules[modname] = importlib.import_module(modname) - except: - skipped_modules.append(modname) + except Exception as e: + skipped_modules[modname] = str(e) api_fqns = [] for modname, module in modules.items(): @@ -62,7 +62,7 @@ def collect_api_entities() -> List[str]: api_fqns.append(f"{modname}.{obj_name}") print() - skipped_str = '\n'.join(skipped_modules) + skipped_str = '\n'.join([f"{k}: {v}" for k, v in skipped_modules.items()]) print(f"Skipped: {skipped_str}\n") print("API entities:") @@ -71,7 +71,7 @@ def collect_api_entities() -> List[str]: return api_fqns -with mock(['torch', 'onnx', 'openvino', 'tensorflow', 'tensorflow_addons']): +with mock(['torch', 'torchvision', 'onnx', 'onnxruntime', 'openvino', 'tensorflow', 'tensorflow_addons']): api_fqns = collect_api_entities() module_fqns = set() diff --git a/nncf/torch/__init__.py b/nncf/torch/__init__.py index d05eef2d30d..67e2cbcfaf4 100644 --- a/nncf/torch/__init__.py +++ b/nncf/torch/__init__.py @@ -17,7 +17,15 @@ import torch from pkg_resources import parse_version -torch_version = parse_version(torch.__version__).base_version + +try: + _torch_version = torch.__version__ + torch_version = parse_version(_torch_version).base_version +except: + nncf_logger.debug("Could not parse torch version") + _torch_version = '0.0.0' + torch_version = parse_version(_torch_version).base_version + if parse_version(BKC_TORCH_VERSION).base_version != torch_version: warn_bkc_version_mismatch("torch", BKC_TORCH_VERSION, torch.__version__) From aa8e7147095e36d5165492fe5940f1c0d1d2b036 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Fri, 14 Apr 2023 11:58:11 +0200 Subject: [PATCH 15/19] Fix pull request targeting --- .github/workflows/api_changes_check.yml | 10 +++++----- .github/workflows/build_schema_page.yml | 5 ----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/api_changes_check.yml b/.github/workflows/api_changes_check.yml index 08e457fdc6d..7fc62ebd78f 100644 --- a/.github/workflows/api_changes_check.yml +++ b/.github/workflows/api_changes_check.yml @@ -1,6 +1,6 @@ name: API changes check on: - pull_request_target: + pull_request: branches: - develop @@ -35,11 +35,11 @@ jobs: echo ${CHANGED_FILES} CHANGED_FILES=$(echo $CHANGED_FILES | tr '\n' ' ') echo "changed_files=${CHANGED_FILES}" >> $GITHUB_OUTPUT - - uses: actions-ecosystem/action-add-labels@v1 - if: ${{ steps.diff.outputs.changed_files }} != "" + - uses: actions-ecosystem/action-remove-labels@v1 + if: ${{ !contains(steps.diff.outputs.changed_files, 'differ') }} with: labels: API - - uses: actions-ecosystem/action-remove-labels@v1 - if: ${{ steps.diff.outputs.changed_files }} == "" + - uses: actions-ecosystem/action-add-labels@v1 + if: ${{ contains(steps.diff.outputs.changed_files, 'differ') }} with: labels: API diff --git a/.github/workflows/build_schema_page.yml b/.github/workflows/build_schema_page.yml index b1368792941..c14b697e642 100644 --- a/.github/workflows/build_schema_page.yml +++ b/.github/workflows/build_schema_page.yml @@ -1,10 +1,5 @@ name: Config schema HTML build on: - pull_request_target: - branches: - - develop - paths: - - nncf/config/** workflow_call: jobs: build-config-schema-html: From 30ab20adb8d1720d8cbb11a81cfad66b6ed7e7e5 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Fri, 14 Apr 2023 12:30:25 +0200 Subject: [PATCH 16/19] Fix derived class handling --- docs/api/source/conf.py | 9 +++++++-- nncf/common/api_marker.py | 4 +++- nncf/common/quantization/structs.py | 1 - nncf/torch/automl/environment/__init__.py | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index d353c2da529..4e1ae63bbf6 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -58,8 +58,13 @@ def collect_api_entities() -> List[str]: if objects_module == modname: if inspect.isclass(obj) or inspect.isfunction(obj): if hasattr(obj, "_nncf_api_marker"): - print(f"\t{obj_name}") - api_fqns.append(f"{modname}.{obj_name}") + marked_object_name = obj._nncf_api_marker + # Check the actual name of the originally marked object + # so that the classes derived from base API classes don't + # all automatically end up in API + if marked_object_name == obj.__name__: + print(f"\t{obj_name}") + api_fqns.append(f"{modname}.{obj_name}") print() skipped_str = '\n'.join([f"{k}: {v}" for k, v in skipped_modules.items()]) diff --git a/nncf/common/api_marker.py b/nncf/common/api_marker.py index 912ee8f3ef3..5961a061476 100644 --- a/nncf/common/api_marker.py +++ b/nncf/common/api_marker.py @@ -5,5 +5,7 @@ def __init__(self): pass def __call__(self, obj): - setattr(obj, api.API_MARKER_ATTR, True) + # The value of the marker will be useful in determining + # whether we are handling a base class or a derived one. + setattr(obj, api.API_MARKER_ATTR, obj.__name__) return obj diff --git a/nncf/common/quantization/structs.py b/nncf/common/quantization/structs.py index 251991b2d46..61f284b21e1 100644 --- a/nncf/common/quantization/structs.py +++ b/nncf/common/quantization/structs.py @@ -15,7 +15,6 @@ from enum import Enum from typing import Dict, List, Optional, Any -from nncf.common.api_marker import api from nncf.common.graph import NNCFNode from nncf.common.graph import NNCFNodeName from nncf.config.schemata.defaults import QUANTIZATION_BITS diff --git a/nncf/torch/automl/environment/__init__.py b/nncf/torch/automl/environment/__init__.py index de0da022181..8727b935935 100644 --- a/nncf/torch/automl/environment/__init__.py +++ b/nncf/torch/automl/environment/__init__.py @@ -9,4 +9,4 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -""" \ No newline at end of file +""" From 79d8f1c6920e12068b7d1f8af5311727dbfbe1a4 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Mon, 17 Apr 2023 14:14:53 +0200 Subject: [PATCH 17/19] Add more API entities --- nncf/data/dataset.py | 2 ++ nncf/parameters.py | 5 +++++ nncf/scopes.py | 12 +++++++----- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/nncf/data/dataset.py b/nncf/data/dataset.py index 8b7ea3f3960..3b974991bce 100644 --- a/nncf/data/dataset.py +++ b/nncf/data/dataset.py @@ -18,11 +18,13 @@ from typing import Generic from typing import TypeVar +from nncf.common.api_marker import api DataItem = TypeVar('DataItem') ModelInput = TypeVar('ModelInput') +@api() class Dataset(Generic[DataItem, ModelInput]): """ The `nncf.Dataset` class defines the interface by which compression algorithms diff --git a/nncf/parameters.py b/nncf/parameters.py index cb4a90cb9b2..d035c7a2b15 100644 --- a/nncf/parameters.py +++ b/nncf/parameters.py @@ -13,6 +13,10 @@ from enum import Enum +from nncf.common.api_marker import api + + +@api() class TargetDevice(Enum): """ Describes the target device the specificity of which will be taken @@ -32,6 +36,7 @@ class TargetDevice(Enum): CPU_SPR = 'CPU_SPR' +@api() class ModelType(Enum): """ Describes the model type the specificity of which will be taken into diff --git a/nncf/scopes.py b/nncf/scopes.py index feff6869c35..6ded64fd3a4 100644 --- a/nncf/scopes.py +++ b/nncf/scopes.py @@ -16,10 +16,12 @@ import re from typing import List, Optional -from nncf.common.graph.graph import NNCFGraph +from nncf.common.api_marker import api from nncf.common.logging import nncf_logger +from nncf.common.graph.graph import NNCFGraph +@api() @dataclass class IgnoredScope: """ @@ -111,11 +113,11 @@ def get_ignored_node_names_from_ignored_scope(ignored_scope: IgnoredScope, if ignored_node_name in node_names: matched_by_names.append(ignored_node_name) if strict and len(ignored_scope.names) != len(matched_by_names): - skipped_names = set(ignored_scope.names) - set(matched_by_names) - raise RuntimeError(f'Ignored nodes with name {list(skipped_names)}' - ' were not found in the NNCFGraph. ' + error_msg) + skipped_names = set(ignored_scope.names) - set(matched_by_names) + raise RuntimeError(f'Ignored nodes with name {list(skipped_names)}' + ' were not found in the NNCFGraph. ' + error_msg) nncf_logger.info(f'{len(matched_by_names)}' - ' ignored nodes was found by name in the NNCFGraph') + ' ignored nodes was found by name in the NNCFGraph') matched_by_patterns = [] if ignored_scope.patterns: From d8a01cc3c609cf62ecd2b41b496e17f87b74cc33 Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Wed, 19 Apr 2023 15:18:16 +0200 Subject: [PATCH 18/19] Support import aliasing --- docs/api/source/conf.py | 34 +++++++++++++++---- nncf/__init__.py | 2 ++ .../accuracy_aware_training/training_loop.py | 3 ++ nncf/common/api_marker.py | 26 ++++++++++++-- nncf/common/quantization/structs.py | 2 ++ nncf/config/config.py | 2 +- nncf/data/dataset.py | 2 +- nncf/parameters.py | 4 +-- nncf/quantization/quantize.py | 3 +- nncf/scopes.py | 2 +- nncf/tensorflow/__init__.py | 1 + nncf/tensorflow/helpers/callback_creation.py | 3 +- nncf/tensorflow/helpers/model_creation.py | 2 +- nncf/tensorflow/initialization.py | 2 +- nncf/torch/checkpoint_loading.py | 3 +- nncf/torch/dynamic_graph/context.py | 4 +++ nncf/torch/dynamic_graph/io_handling.py | 3 ++ nncf/torch/dynamic_graph/patch_pytorch.py | 2 ++ nncf/torch/extensions/__init__.py | 3 ++ nncf/torch/initialization.py | 2 +- nncf/torch/layers.py | 2 ++ nncf/torch/model_creation.py | 2 +- 22 files changed, 89 insertions(+), 20 deletions(-) diff --git a/docs/api/source/conf.py b/docs/api/source/conf.py index 4e1ae63bbf6..ccb44a06659 100644 --- a/docs/api/source/conf.py +++ b/docs/api/source/conf.py @@ -3,6 +3,7 @@ import os import pkgutil import sys +from typing import Dict from typing import List from sphinx.ext.autodoc import mock @@ -50,30 +51,51 @@ def collect_api_entities() -> List[str]: except Exception as e: skipped_modules[modname] = str(e) - api_fqns = [] + from nncf.common.api_marker import api + api_fqns = dict() + aliased_fqns = {} # type: Dict[str, str] for modname, module in modules.items(): print(f"{modname}") for obj_name, obj in inspect.getmembers(module): objects_module = getattr(obj, '__module__', None) if objects_module == modname: if inspect.isclass(obj) or inspect.isfunction(obj): - if hasattr(obj, "_nncf_api_marker"): + if hasattr(obj, api.API_MARKER_ATTR): marked_object_name = obj._nncf_api_marker # Check the actual name of the originally marked object # so that the classes derived from base API classes don't # all automatically end up in API - if marked_object_name == obj.__name__: - print(f"\t{obj_name}") - api_fqns.append(f"{modname}.{obj_name}") + if marked_object_name != obj.__name__: + continue + fqn = f"{modname}.{obj_name}" + if hasattr(obj, api.CANONICAL_ALIAS_ATTR): + canonical_import_name = getattr(obj, api.CANONICAL_ALIAS_ATTR) + aliased_fqns[fqn] = canonical_import_name + if canonical_import_name == fqn: + print(f"\t{obj_name}") + else: + print(f"\t{obj_name} -> {canonical_import_name}") + api_fqns[fqn] = True print() skipped_str = '\n'.join([f"{k}: {v}" for k, v in skipped_modules.items()]) print(f"Skipped: {skipped_str}\n") + for fqn, canonical_alias in aliased_fqns.items(): + try: + module_name, _, function_name = canonical_alias.rpartition('.') + getattr(importlib.import_module(module_name), function_name) + except (ImportError, AttributeError) as e: + print( + f"API entity with canonical_alias={canonical_alias} not available for import as specified!\n" + f"Adjust the __init__.py files so that the symbol is available for import as {canonical_alias}.") + raise e + api_fqns.pop(fqn) + api_fqns[canonical_alias] = True print("API entities:") for api_fqn in api_fqns: print(api_fqn) - return api_fqns + return list(api_fqns.keys()) with mock(['torch', 'torchvision', 'onnx', 'onnxruntime', 'openvino', 'tensorflow', 'tensorflow_addons']): diff --git a/nncf/__init__.py b/nncf/__init__.py index b904ddf36fc..79a6b67d36d 100644 --- a/nncf/__init__.py +++ b/nncf/__init__.py @@ -60,3 +60,5 @@ else: nncf_logger.info(f"NNCF initialized successfully. Supported frameworks detected: " f"{', '.join([name for name, loaded in _LOADED_FRAMEWORKS.items() if loaded])}") + + diff --git a/nncf/common/accuracy_aware_training/training_loop.py b/nncf/common/accuracy_aware_training/training_loop.py index 6a640a5a2f8..53a1f63a555 100644 --- a/nncf/common/accuracy_aware_training/training_loop.py +++ b/nncf/common/accuracy_aware_training/training_loop.py @@ -20,6 +20,7 @@ from scipy.interpolate import interp1d from nncf.api.compression import CompressionAlgorithmController +from nncf.common.api_marker import api from nncf.common.composite_compression import CompositeCompressionAlgorithmController from nncf.common.logging import nncf_logger from nncf.common.utils.registry import Registry @@ -168,6 +169,7 @@ def _accuracy_criterion_satisfied(self): return accuracy_budget >= 0 and self.runner.is_model_fully_compressed(self.compression_controller) +@api(canonical_alias="nncf.tensorflow.EarlyExitCompressionTrainingLoop") class EarlyExitCompressionTrainingLoop(BaseEarlyExitCompressionTrainingLoop): """ Adaptive compression training loop allows an accuracy-aware training process @@ -191,6 +193,7 @@ def __init__(self, self.runner = runner_factory.create_training_loop() +@api(canonical_alias="nncf.tensorflow.AdaptiveCompressionTrainingLoop") class AdaptiveCompressionTrainingLoop(BaseEarlyExitCompressionTrainingLoop): """ Adaptive compression training loop allows an accuracy-aware training process whereby diff --git a/nncf/common/api_marker.py b/nncf/common/api_marker.py index 5961a061476..c8939da9d89 100644 --- a/nncf/common/api_marker.py +++ b/nncf/common/api_marker.py @@ -1,11 +1,33 @@ +""" + Copyright (c) 2023 Intel Corporation + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + + class api: API_MARKER_ATTR = "_nncf_api_marker" + CANONICAL_ALIAS_ATTR = "_nncf_canonical_alias" - def __init__(self): - pass + def __init__(self, canonical_alias: str = None): + self._canonical_alias = canonical_alias def __call__(self, obj): # The value of the marker will be useful in determining # whether we are handling a base class or a derived one. setattr(obj, api.API_MARKER_ATTR, obj.__name__) + if self._canonical_alias is not None: + setattr(obj, api.CANONICAL_ALIAS_ATTR, self._canonical_alias) return obj + + +def is_api(obj) -> bool: + return hasattr(obj, api.API_MARKER_ATTR) + diff --git a/nncf/common/quantization/structs.py b/nncf/common/quantization/structs.py index 61f284b21e1..4988055688a 100644 --- a/nncf/common/quantization/structs.py +++ b/nncf/common/quantization/structs.py @@ -15,6 +15,7 @@ from enum import Enum from typing import Dict, List, Optional, Any +from nncf.common.api_marker import api from nncf.common.graph import NNCFNode from nncf.common.graph import NNCFNodeName from nncf.config.schemata.defaults import QUANTIZATION_BITS @@ -308,6 +309,7 @@ class UnifiedScaleType(Enum): UNIFY_ALWAYS = 1 +@api(canonical_alias="nncf.QuantizationPreset") class QuantizationPreset(Enum): PERFORMANCE = 'performance' MIXED = 'mixed' diff --git a/nncf/config/config.py b/nncf/config/config.py index 68a8d3ffae7..1cf960f6a41 100644 --- a/nncf/config/config.py +++ b/nncf/config/config.py @@ -30,7 +30,7 @@ from nncf.config.structures import NNCFExtraConfigStruct -@api() +@api(canonical_alias="nncf.NNCFConfig") class NNCFConfig(dict): """A regular dictionary object extended with some utility functions.""" diff --git a/nncf/data/dataset.py b/nncf/data/dataset.py index 3b974991bce..1d7ccffe1c4 100644 --- a/nncf/data/dataset.py +++ b/nncf/data/dataset.py @@ -24,7 +24,7 @@ ModelInput = TypeVar('ModelInput') -@api() +@api(canonical_alias="nncf.Dataset") class Dataset(Generic[DataItem, ModelInput]): """ The `nncf.Dataset` class defines the interface by which compression algorithms diff --git a/nncf/parameters.py b/nncf/parameters.py index d035c7a2b15..083fd9f43bf 100644 --- a/nncf/parameters.py +++ b/nncf/parameters.py @@ -16,7 +16,7 @@ from nncf.common.api_marker import api -@api() +@api(canonical_alias="nncf.TargetDevice") class TargetDevice(Enum): """ Describes the target device the specificity of which will be taken @@ -36,7 +36,7 @@ class TargetDevice(Enum): CPU_SPR = 'CPU_SPR' -@api() +@api(canonical_alias="nncf.ModelType") class ModelType(Enum): """ Describes the model type the specificity of which will be taken into diff --git a/nncf/quantization/quantize.py b/nncf/quantization/quantize.py index c3e95ac0870..a0e3a3748d0 100644 --- a/nncf/quantization/quantize.py +++ b/nncf/quantization/quantize.py @@ -27,7 +27,7 @@ from nncf.parameters import TargetDevice -@api() +@api(canonical_alias="nncf.quantize") def quantize(model: TModel, calibration_dataset: Dataset, preset: QuantizationPreset = QuantizationPreset.PERFORMANCE, @@ -85,6 +85,7 @@ def quantize(model: TModel, raise RuntimeError(f'Unsupported type of backend: {backend}') +@api(canonical_alias="nncf.quantize_with_accuracy_control") def quantize_with_accuracy_control(model: TModel, calibration_dataset: Dataset, validation_dataset: Dataset, diff --git a/nncf/scopes.py b/nncf/scopes.py index 6ded64fd3a4..160fcbd4491 100644 --- a/nncf/scopes.py +++ b/nncf/scopes.py @@ -21,7 +21,7 @@ from nncf.common.graph.graph import NNCFGraph -@api() +@api(canonical_alias="nncf.IgnoredScope") @dataclass class IgnoredScope: """ diff --git a/nncf/tensorflow/__init__.py b/nncf/tensorflow/__init__.py index bd3eb2bb55a..fa8793b0417 100644 --- a/nncf/tensorflow/__init__.py +++ b/nncf/tensorflow/__init__.py @@ -19,6 +19,7 @@ from pkg_resources import parse_version try: + _tf_version = tensorflow.__version__ tensorflow_version = parse_version(_tf_version).base_version except: nncf_logger.debug("Could not parse tensorflow version") diff --git a/nncf/tensorflow/helpers/callback_creation.py b/nncf/tensorflow/helpers/callback_creation.py index 9488bd037d3..ad1280aedcd 100644 --- a/nncf/tensorflow/helpers/callback_creation.py +++ b/nncf/tensorflow/helpers/callback_creation.py @@ -10,7 +10,7 @@ See the License for the specific language governing permissions and limitations under the License. """ - +from nncf.common.api_marker import api from nncf.common.composite_compression import CompositeCompressionAlgorithmController from nncf.tensorflow.pruning.base_algorithm import BasePruningAlgoController from nncf.tensorflow.pruning.callbacks import PruningStatisticsCallback @@ -19,6 +19,7 @@ from nncf.tensorflow.sparsity.base_algorithm import BaseSparsityController +@api(canonical_alias="nncf.tensorflow.create_compression_callbacks") def create_compression_callbacks(compression_ctrl, log_tensorboard=True, log_text=True, log_dir=None): compression_controllers = compression_ctrl.child_ctrls \ if isinstance(compression_ctrl, CompositeCompressionAlgorithmController) \ diff --git a/nncf/tensorflow/helpers/model_creation.py b/nncf/tensorflow/helpers/model_creation.py index 2322d18c0ae..4ab46cb849f 100644 --- a/nncf/tensorflow/helpers/model_creation.py +++ b/nncf/tensorflow/helpers/model_creation.py @@ -59,7 +59,7 @@ def create_compression_algorithm_builder(config: NNCFConfig, return TFCompositeCompressionAlgorithmBuilder(config, should_init) -@api() +@api(canonical_alias="nncf.tensorflow.create_compressed_model") @tracked_function(NNCF_TF_CATEGORY, [CompressionStartedFromConfig(argname="config"), ]) def create_compressed_model(model: tf.keras.Model, config: NNCFConfig, diff --git a/nncf/tensorflow/initialization.py b/nncf/tensorflow/initialization.py index 7f6a89224e9..499b97a795d 100644 --- a/nncf/tensorflow/initialization.py +++ b/nncf/tensorflow/initialization.py @@ -39,7 +39,7 @@ def __iter__(self): return iter(self._data_loader) -@api() +@api(canonical_alias="nncf.tensorflow.register_default_init_args") def register_default_init_args(nncf_config: NNCFConfig, data_loader: tf.data.Dataset, batch_size: int, diff --git a/nncf/torch/checkpoint_loading.py b/nncf/torch/checkpoint_loading.py index 6ca6a4095e9..be563f1b300 100644 --- a/nncf/torch/checkpoint_loading.py +++ b/nncf/torch/checkpoint_loading.py @@ -16,10 +16,11 @@ import torch +from nncf.common.api_marker import api from nncf.common.logging import nncf_logger from nncf.common.deprecation import warning_deprecated - +@api(canonical_alias="nncf.torch.load_state") def load_state(model: torch.nn.Module, state_dict_to_load: dict, is_resume: bool = False, keys_to_ignore: List[str] = None) -> int: """ diff --git a/nncf/torch/dynamic_graph/context.py b/nncf/torch/dynamic_graph/context.py index c5069a8fa4e..e3ab86a87b9 100644 --- a/nncf/torch/dynamic_graph/context.py +++ b/nncf/torch/dynamic_graph/context.py @@ -22,6 +22,7 @@ import torch +from nncf.common.api_marker import api from nncf.common.graph.layer_attributes import BaseLayerAttributes from nncf.common.utils.debug import is_debug from nncf.torch.dynamic_graph.graph import DynamicGraph @@ -429,6 +430,7 @@ def set_current_context(c: TracingContext): _CURRENT_CONTEXT.context = c +@api(canonical_alias="nncf.torch.no_nncf_trace") @contextmanager def no_nncf_trace(): ctx = get_current_context() @@ -440,6 +442,7 @@ def no_nncf_trace(): yield +@api(canonical_alias="nncf.torch.forward_nncf_trace") @contextmanager def forward_nncf_trace(): ctx = get_current_context() @@ -455,6 +458,7 @@ def get_current_context() -> TracingContext: return _CURRENT_CONTEXT.context +@api(canonical_alias="nncf.torch.disable_tracing") def disable_tracing(method): """ Patch a method so that it will be executed within no_nncf_trace context diff --git a/nncf/torch/dynamic_graph/io_handling.py b/nncf/torch/dynamic_graph/io_handling.py index 51e50b1815d..16da857978a 100644 --- a/nncf/torch/dynamic_graph/io_handling.py +++ b/nncf/torch/dynamic_graph/io_handling.py @@ -5,6 +5,7 @@ import torch +from nncf.common.api_marker import api from nncf.common.graph.definitions import MODEL_INPUT_OP_NAME from nncf.common.graph.definitions import MODEL_OUTPUT_OP_NAME from nncf.torch.dynamic_graph.patch_pytorch import register_operator @@ -16,11 +17,13 @@ from nncf.torch.dynamic_graph.context import forward_nncf_trace +@api(canonical_alias="nncf.torch.nncf_model_input") @register_operator(name=MODEL_INPUT_OP_NAME) def nncf_model_input(tensor: 'torch.Tensor'): return tensor +@api(canonical_alias="nncf.torch.nncf_model_output") @register_operator(name=MODEL_OUTPUT_OP_NAME) def nncf_model_output(tensor: 'torch.Tensor'): return tensor diff --git a/nncf/torch/dynamic_graph/patch_pytorch.py b/nncf/torch/dynamic_graph/patch_pytorch.py index cace2681f7a..e85e8d085f3 100644 --- a/nncf/torch/dynamic_graph/patch_pytorch.py +++ b/nncf/torch/dynamic_graph/patch_pytorch.py @@ -24,6 +24,7 @@ from torch.nn.parallel import DistributedDataParallel from nncf import nncf_logger +from nncf.common.api_marker import api from nncf.torch.dynamic_graph.structs import NamespaceTarget from nncf.torch.dynamic_graph.trace_tensor import TracedTensor from nncf.torch.dynamic_graph.wrappers import ignore_scope @@ -96,6 +97,7 @@ class MagicFunctionsToPatch: } +@api(canonical_alias="nncf.torch.load_state") def register_operator(name=None): def wrap(operator): op_name = name diff --git a/nncf/torch/extensions/__init__.py b/nncf/torch/extensions/__init__.py index b7c40e866d5..7a014ade93a 100644 --- a/nncf/torch/extensions/__init__.py +++ b/nncf/torch/extensions/__init__.py @@ -8,6 +8,7 @@ from torch.utils.cpp_extension import _get_build_directory +from nncf.common.api_marker import api from nncf.common.logging.logger import extension_is_loading_info_log from nncf.common.utils.registry import Registry from nncf.common.logging import nncf_logger @@ -80,10 +81,12 @@ def _force_build_extensions(ext_type: ExtensionsType): class_type.load() +@api(canonical_alias="nncf.torch.force_build_cpu_extensions") def force_build_cpu_extensions(): _force_build_extensions(ExtensionsType.CPU) +@api(canonical_alias="nncf.torch.force_build_cuda_extensions") def force_build_cuda_extensions(): _force_build_extensions(ExtensionsType.CUDA) diff --git a/nncf/torch/initialization.py b/nncf/torch/initialization.py index 42bbebcb736..f673581e5c4 100644 --- a/nncf/torch/initialization.py +++ b/nncf/torch/initialization.py @@ -225,7 +225,7 @@ def default_criterion_fn(outputs: Any, target: Any, criterion: Any) -> torch.Ten return criterion(outputs, target) -@api() +@api(canonical_alias="nncf.torch.register_default_init_args") def register_default_init_args(nncf_config: 'NNCFConfig', train_loader: torch.utils.data.DataLoader, criterion: _Loss = None, diff --git a/nncf/torch/layers.py b/nncf/torch/layers.py index c938ad6467d..8f5a1c9423c 100644 --- a/nncf/torch/layers.py +++ b/nncf/torch/layers.py @@ -25,6 +25,7 @@ from torch.nn.utils.weight_norm import WeightNorm from nncf import nncf_logger +from nncf.common.api_marker import api from nncf.torch.dynamic_graph.context import forward_nncf_trace from nncf.torch.utils import no_jit_trace from nncf.torch.checkpoint_loading import OPTIONAL_PARAMETERS_REGISTRY @@ -385,6 +386,7 @@ def from_module(module): NNCF_WRAPPED_USER_MODULES_DICT = {} +@api(canonical_alias="nncf.torch.load_state") def register_module(*quantizable_field_names: str, ignored_algorithms: list = None): # quantizable_field_names will work for `weight` attributes only. Should later extend to registering # customly named attributes if it becomes necessary diff --git a/nncf/torch/model_creation.py b/nncf/torch/model_creation.py index 275de53dc65..1d663e06f28 100644 --- a/nncf/torch/model_creation.py +++ b/nncf/torch/model_creation.py @@ -45,7 +45,7 @@ from nncf.torch.utils import training_mode_switcher -@api() +@api(canonical_alias="nncf.torch.create_compressed_model") @tracked_function(NNCF_PT_CATEGORY, [CompressionStartedFromConfig(argname="config"), ]) def create_compressed_model(model: Module, config: NNCFConfig, From 52462139dccbae80b2cb8e0e4656d2b219ef491a Mon Sep 17 00:00:00 2001 From: Vasily Shamporov Date: Thu, 13 Apr 2023 16:45:01 +0200 Subject: [PATCH 19/19] Non-API change --- nncf/torch/model_creation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nncf/torch/model_creation.py b/nncf/torch/model_creation.py index 1d663e06f28..1e36de2259b 100644 --- a/nncf/torch/model_creation.py +++ b/nncf/torch/model_creation.py @@ -91,6 +91,7 @@ def create_compressed_model(model: Module, as an object of NNCFNetwork.""" set_debug_log_dir(config.get("log_dir", ".")) + print("Test") is_legacy_model_state_dict = compression_state is not None and \ BaseController.BUILDER_STATE not in compression_state and \