Merge pull request #4207 from whisperity/chore/config/documentation-u…

…rls-for-gone-checkers

feat(script): Verify the existence of checker config `doc_url` pages and find appropriate older releases for gone (removed, dealpha, etc.) checkers

codechecker_common/checker_labels.py

@@ -13,6 +13,22 @@
 from codechecker_common.util import load_json
 
 
+def split_label_kv(key_value: str) -> Tuple[str, str]:
+    """
+    A label has a value separated by colon (:) character, e.g:
+    "severity:high". This function returns this key and value as a tuple.
+    Optional whitespaces around (:) and at the two ends of this string are
+    not taken into account. If key_value contains no colon, then the value
+    is empty string.
+    """
+    try:
+        pos = key_value.index(':')
+    except ValueError:
+        return (key_value.strip(), '')
+
+    return key_value[:pos].strip(), key_value[pos + 1:].strip()
+
+
 # TODO: Most of the methods of this class get an optional analyzer name. If
 # None is given to these functions then labels of any analyzer's checkers is
 # taken into account. This the union of all analyzers' checkers is a bad
@@ -79,21 +95,6 @@ def __union_label_files(
 
         return all_labels
 
-    def __get_label_key_value(self, key_value: str) -> Tuple[str, str]:
-        """
-        A label has a value separated by colon (:) character, e.g:
-        "severity:high". This function returns this key and value as a tuple.
-        Optional whitespaces around (:) and at the two ends of this string are
-        not taken into account. If key_value contains no colon, then the value
-        is empty string.
-        """
-        try:
-            pos = key_value.index(':')
-        except ValueError:
-            return (key_value.strip(), '')
-
-        return key_value[:pos].strip(), key_value[pos + 1:].strip()
-
     def __check_json_format(self, data: dict):
         """
         Check the format of checker labels' JSON config file, i.e. this file
@@ -111,7 +112,7 @@ def is_unique(labels: Iterable[str], label: str):
             Check if the given label occurs only once in the label list.
             """
             found = False
-            for k, _ in map(self.__get_label_key_value, labels):
+            for k, _ in map(split_label_kv, labels):
                 if k == label:
                     if found:
                         return False
@@ -173,11 +174,11 @@ def checkers_by_labels(
         """
         collection = []
 
-        label_set = set(map(self.__get_label_key_value, filter_labels))
+        label_set = set(map(split_label_kv, filter_labels))
 
         for _, checkers in self.__get_analyzer_data(analyzer):
             for checker, labels in checkers.items():
-                labels = set(map(self.__get_label_key_value, labels))
+                labels = set(map(split_label_kv, labels))
 
                 if labels.intersection(label_set):
                     collection.append(checker)
@@ -243,8 +244,7 @@ def labels_of_checker(
                     lambda c: checker.startswith(cast(str, c)),
                     iter(checkers.keys())), None)
 
-            labels.extend(
-                map(self.__get_label_key_value, checkers.get(c, [])))
+            labels.extend(map(split_label_kv, checkers.get(c, [])))
 
         # TODO set() is used for uniqueing results in case a checker name is
         # provided by multiple analyzers. This will be unnecessary when we
@@ -277,7 +277,7 @@ def labels(self, analyzer: Optional[str] = None) -> List[str]:
         for _, checkers in self.__get_analyzer_data(analyzer):
             for labels in checkers.values():
                 collection.update(map(
-                    lambda x: self.__get_label_key_value(x)[0], labels))
+                    lambda x: split_label_kv(x)[0], labels))
 
         return list(collection)
 
@@ -294,7 +294,7 @@ def occurring_values(
 
         for _, checkers in self.__get_analyzer_data(analyzer):
             for labels in checkers.values():
-                for lab, value in map(self.__get_label_key_value, labels):
+                for lab, value in map(split_label_kv, labels):
                     if lab == label:
                         values.add(value)
 

codechecker_common/util.py

@@ -34,9 +34,7 @@ def arg_match(options, args):
 
 
 def clamp(min_: int, value: int, max_: int) -> int:
-    """
-    Clamps ``value`` to be between ``min_`` and ``max_``, inclusive.
-    """
+    """Clamps ``value`` such that ``min_ <= value <= max_``."""
     if min_ > max_:
         raise ValueError("min <= max required")
     return min(max(min_, value), max_)

scripts/labels/compiler_warnings.py

@@ -1,3 +1,4 @@
+# FIXME: Subsume into the newer label_tool package.
 import argparse
 import json
 import urllib3

scripts/labels/doc_url.py → scripts/labels/doc_url_generate.py

@@ -1,3 +1,4 @@
+# FIXME: Subsume into the newer label_tool/doc_url package!
 import argparse
 import json
 import sys

scripts/labels/label_tool/__init__.py

@@ -0,0 +1,31 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+"""
+This library ships reusable components and user-facing tools to verify,
+generate, and adapt the checker labels in the CodeChecker configuration
+structure.
+"""
+# Load the interpreter injection first.
+from . import codechecker
+
+from . import \
+    checker_labels, \
+    http_, \
+    output, \
+    transformer, \
+    util
+
+
+__all__ = [
+    "checker_labels",
+    "codechecker",
+    "http_",
+    "output",
+    "transformer",
+    "util",
+]

scripts/labels/label_tool/__main__.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+"""Dispatching to the top-level tools implemented in the package."""
+import argparse
+import sys
+
+
+try:
+    from .doc_url.verify_tool import __main__ as doc_url_verify
+except ModuleNotFoundError as e:
+    import traceback
+    traceback.print_exc()
+
+    print("\nFATAL: Failed to import some required modules! "
+          "Please make sure you also install the contents of the "
+          "'requirements.txt' of this tool into your virtualenv:\n"
+          "\tpip install -r scripts/requirements.txt",
+          file=sys.stderr)
+    sys.exit(1)
+
+
+def args() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=__package__,
+        description="""
+Tooling related to creating, managing, verifying, and updating the checker
+labels in a CodeChecker config directory.
+This main script is the union of several independent tools using a common
+internal library.
+""",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    subparsers = parser.add_subparsers(
+        title="subcommands",
+        description="Please select a subcommand to continue.",
+        dest="subcommand",
+        required=True)
+
+    def add_subparser(command: str, package):
+        subparser = subparsers.add_parser(
+            command,
+            prog=package.__package__,
+            help=package.short_help,
+            description=package.description,
+            epilog=package.epilogue,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        subparser = package.args(subparser)
+        subparser.set_defaults(__main=package.main)
+
+    add_subparser("doc_url_verify", doc_url_verify)
+
+    return parser
+
+
+if __name__ == "__main__":
+    def _main():
+        _args = args().parse_args()
+        del _args.__dict__["subcommand"]
+
+        main = _args.__dict__["__main"]
+        del _args.__dict__["__main"]
+
+        sys.exit(main(_args) or 0)
+    _main()

scripts/labels/label_tool/checker_labels.py

@@ -0,0 +1,140 @@
+# -------------------------------------------------------------------------
+#
+#  Part of the CodeChecker project, under the Apache License v2.0 with
+#  LLVM Exceptions. See LICENSE for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# -------------------------------------------------------------------------
+"""Provides I/O with the configuration files that describe checker labels."""
+from collections import deque
+import json
+import pathlib
+from typing import Dict, List, Optional, cast
+
+from codechecker_common.checker_labels import split_label_kv
+
+from .output import Settings as OutputSettings, error, trace
+
+
+_ConfigFileLabels = Dict[str, List[str]]
+
+SingleLabels = Dict[str, Optional[str]]
+Labels = Dict[str, Dict[str, str]]
+
+
+def _load_json(path: pathlib.Path) -> Dict:
+    try:
+        with path.open("r") as file:
+            return json.load(file)
+    except OSError:
+        import traceback
+        traceback.print_exc()
+
+        error("Failed to open label config file '%s'", path)
+        raise
+    except json.JSONDecodeError:
+        import traceback
+        traceback.print_exc()
+
+        error("Failed to parse label config file '%s'", path)
+        raise
+
+
+def _save_json(path: pathlib.Path, data: Dict):
+    try:
+        with path.open("w") as file:
+            json.dump(data, file, indent=2)
+            file.write('\n')
+    except OSError:
+        import traceback
+        traceback.print_exc()
+
+        error("Failed to write label config file '%s'", path)
+        raise
+    except (TypeError, ValueError):
+        import traceback
+        traceback.print_exc()
+
+        error("Failed to encode label config file '%s'", path)
+        raise
+
+
+class MultipleLabelsError(Exception):
+    """
+    Raised by `get_checker_labels` if multiple labels exist for the same key.
+    """
+
+    def __init__(self, key):
+        super().__init__("Multiple labels with key: %s", key)
+        self.key = key
+
+
+def get_checker_labels(analyser: str, path: pathlib.Path, key: str) \
+        -> SingleLabels:
+    """
+    Loads and filters the checker config label file available at `path`
+    for the `key` label. Raises `MultipleLabelsError` if there is at least
+    two labels with the same `key`.
+    """
+    try:
+        label_cfg = cast(_ConfigFileLabels, _load_json(path)["labels"])
+    except KeyError:
+        error("'%s' is not a label config file", path)
+        raise
+
+    filtered_labels = {
+        checker: [label_v
+                  for label in labels
+                  for label_k, label_v in (split_label_kv(label),)
+                  if label_k == key]
+        for checker, labels in label_cfg.items()}
+    if OutputSettings.trace():
+        deque((trace("No '%s:' label found for '%s/%s'",
+                     key, analyser, checker)
+               for checker, labels in filtered_labels.items()
+               if not labels), maxlen=0)
+
+    if any(len(labels) > 1 for labels in filtered_labels.values()):
+        raise MultipleLabelsError(key)
+    return {checker: labels[0] if labels else None
+            for checker, labels in filtered_labels.items()}
+
+
+def update_checker_labels(analyser: str,
+                          path: pathlib.Path,
+                          key: str,
+                          updates: SingleLabels):
+    """
+    Loads a checker config label file available at `path` and updates the
+    `key` labels based on the `updates` structure, overwriting or adding the
+    existing label (or raising `MultipleLabelsError` if it is not unique which
+    one to overwrite), then writes the resulting data structure back to `path`.
+    """
+    try:
+        config = _load_json(path)
+        label_cfg = cast(_ConfigFileLabels, config["labels"])
+    except KeyError:
+        error("'%s's '%s' is not a label config file", analyser, path)
+        raise
+
+    label_indices = {
+        checker: [index for index, label in enumerate(labels)
+                  if split_label_kv(label)[0] == key]
+        for checker, labels in label_cfg.items()
+    }
+
+    if any(len(indices) > 1 for indices in label_indices.values()):
+        raise MultipleLabelsError(key)
+    label_indices = {checker: indices[0] if len(indices) == 1 else None
+                     for checker, indices in label_indices.items()}
+    for checker, new_label in updates.items():
+        checker_labels = label_cfg[checker]
+        idx = label_indices[checker]
+        e = f"{key}:{new_label}"
+        if idx is not None:
+            checker_labels[idx] = e
+        else:
+            checker_labels.insert(0, e)
+            label_cfg[checker] = sorted(checker_labels)
+
+    _save_json(path, config)