From 5be7ae1d1e60dd0e2b2cc81a87dfb0f3179eb2b2 Mon Sep 17 00:00:00 2001 From: Torgil Svensson Date: Mon, 2 Jan 2023 13:52:44 +0100 Subject: [PATCH] Update ctexplain to run with bazel and newer python versions Use "bazel" instead of "blaze" as default binary. Add option to specify bazel binary. Detect label arguments that starts with "@". Import bazel_api, lib, util and summary from qualified path to avoid collisions for users using this as a module. It also requires fewer entries in PYTHONPATH. Updated to run with Python 3.10. "Mapping" has been moved from collections to collections.abc. Use regular expression instead of split on space to cope with spaces in label name for cquery output. --- .../py/frozendict/frozendict/__init__.py | 6 ++- tools/ctexplain/analyses/summary.py | 2 + tools/ctexplain/bazel_api.py | 37 ++++++++++--------- tools/ctexplain/ctexplain.py | 16 ++++++-- tools/ctexplain/lib.py | 1 + 5 files changed, 40 insertions(+), 22 deletions(-) diff --git a/third_party/py/frozendict/frozendict/__init__.py b/third_party/py/frozendict/frozendict/__init__.py index 399948a61b597b..cf51bcbbef40a8 100644 --- a/third_party/py/frozendict/frozendict/__init__.py +++ b/third_party/py/frozendict/frozendict/__init__.py @@ -9,11 +9,15 @@ except ImportError: # python < 2.7 OrderedDict = NotImplemented +try: + from collections import Mapping +except ImportError: + from collections.abc import Mapping iteritems = getattr(dict, 'iteritems', dict.items) # py2-3 compatibility -class frozendict(collections.Mapping): +class frozendict(Mapping): """ An immutable wrapper around dictionaries that implements the complete :py:class:`collections.Mapping` interface. It can be used as a drop-in replacement for dictionaries where immutability is desired. diff --git a/tools/ctexplain/analyses/summary.py b/tools/ctexplain/analyses/summary.py index 6669e6c5216445..e99a582016c13c 100644 --- a/tools/ctexplain/analyses/summary.py +++ b/tools/ctexplain/analyses/summary.py @@ -20,6 +20,8 @@ from tools.ctexplain.types import ConfiguredTarget # Do not edit this line. Copybara replaces it with PY2 migration helper..third_party.bazel.tools.ctexplain.util as util +import tools.ctexplain.util as util + @dataclass(frozen=True) class _Summary(): diff --git a/tools/ctexplain/bazel_api.py b/tools/ctexplain/bazel_api.py index 02a16174a6eb02..9d78986a56c664 100644 --- a/tools/ctexplain/bazel_api.py +++ b/tools/ctexplain/bazel_api.py @@ -15,8 +15,10 @@ There's no Python Bazel API so we invoke Bazel as a subprocess. """ +import functools import json import os +import re import subprocess from typing import Callable from typing import List @@ -28,8 +30,12 @@ from tools.ctexplain.types import HostConfiguration from tools.ctexplain.types import NullConfiguration +CQUERY_RESULT_LINE_REGEX = re.compile(r"^(.*?) \((\S+)\) \[([^[]*)\]$") +DEFAULT_BAZEL_BINARY = "bazel" -def run_bazel_in_client(args: List[str]) -> Tuple[int, List[str], List[str]]: + +def run_bazel_in_client(args: List[str], + bazel: str = DEFAULT_BAZEL_BINARY) -> Tuple[int, List[str], List[str]]: """Calls bazel within the current workspace. For production use. Tests use an alternative invoker that goes through test @@ -42,7 +48,7 @@ def run_bazel_in_client(args: List[str]) -> Tuple[int, List[str], List[str]]: Tuple of (return code, stdout, stderr) """ result = subprocess.run( - ["blaze"] + args, + [bazel] + args, cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -57,8 +63,12 @@ class BazelApi(): def __init__(self, run_bazel: Callable[[List[str]], Tuple[int, List[str], - List[str]]] = run_bazel_in_client): + List[str]]] = None, + bazel: str = DEFAULT_BAZEL_BINARY): + self.bazel = bazel self.run_bazel = run_bazel + if run_bazel is None: + self.run_bazel = functools.partial(run_bazel_in_client, bazel=self.bazel) def cquery(self, args: List[str]) -> Tuple[bool, str, Tuple[ConfiguredTarget, ...]]: @@ -144,20 +154,13 @@ def _parse_cquery_result_line(line: str) -> ConfiguredTarget: Returns: Corresponding ConfiguredTarget if the line matches else None. """ - tokens = line.split(maxsplit=2) - label = tokens[0] - if tokens[1][0] != "(" or tokens[1][-1] != ")": - raise ValueError(f"{tokens[1]} in {line} not surrounded by parentheses") - config_hash = tokens[1][1:-1] - if config_hash == "null": - fragments = () - else: - if tokens[2][0] != "[" or tokens[2][-1] != "]": - raise ValueError(f"{tokens[2]} in {line} not surrounded by [] brackets") - # The fragments list looks like '[Fragment1, Fragment2, ...]'. Split the - # whole line on ' [' to get just this list, then remove the final ']', then - # split again on ', ' to convert it to a structured tuple. - fragments = tuple(line.split(" [")[1][0:-1].split(", ")) + result = CQUERY_RESULT_LINE_REGEX.search(line) + if result is None: + raise ValueError(f"{repr(line)} does not match {repr(CQUERY_RESULT_LINE_REGEX.pattern)}") + label, config_hash, fragments_str = result.groups() + # The fragments list looks like 'Fragment1, Fragment2, ...'. Split on + # ', ' to convert it to a structured tuple. + fragments = tuple(fragments_str.split(", ")) return ConfiguredTarget( label=label, config=None, # Not yet available: we'll need `bazel config` to get this. diff --git a/tools/ctexplain/ctexplain.py b/tools/ctexplain/ctexplain.py index 65814799a35cb1..4798a943ff0594 100644 --- a/tools/ctexplain/ctexplain.py +++ b/tools/ctexplain/ctexplain.py @@ -43,11 +43,15 @@ from dataclasses import dataclass # Do not edit this line. Copybara replaces it with PY2 migration helper..third_party.bazel.tools.ctexplain.analyses.summary as summary -from tools.ctexplain.bazel_api import BazelApi +from tools.ctexplain.bazel_api import DEFAULT_BAZEL_BINARY, BazelApi # Do not edit this line. Copybara replaces it with PY2 migration helper..third_party.bazel.tools.ctexplain.lib as lib from tools.ctexplain.types import ConfiguredTarget # Do not edit this line. Copybara replaces it with PY2 migration helper..third_party.bazel.tools.ctexplain.util as util +import tools.ctexplain.lib as lib +import tools.ctexplain.util as util +import tools.ctexplain.analyses.summary as summary + FLAGS = flags.FLAGS @@ -112,6 +116,8 @@ def _render_analysis_help_text() -> str: lambda flag_value: all(name in analyses for name in flag_value), message=f'available analyses: {", ".join(analyses.keys())}') +flags.DEFINE_string("bazel", DEFAULT_BAZEL_BINARY, "Path to bazel binary") + flags.DEFINE_multi_string( "build", [], """command-line invocation of the build to analyze. For example: @@ -133,8 +139,8 @@ def _get_build_flags(cmdline: str) -> Tuple[Tuple[str, ...], Tuple[str, ...]]: Tuple of ((target labels to build), (build flags)) """ cmdlist = cmdline.split() - labels = [arg for arg in cmdlist if arg.startswith("//")] - build_flags = [arg for arg in cmdlist if not arg.startswith("//")] + labels = [arg for arg in cmdlist if arg.startswith("//") or arg.startswith("@")] + build_flags = [arg for arg in cmdlist if not arg.startswith("//") and not arg.startswith("@")] return (tuple(labels), tuple(build_flags)) @@ -148,8 +154,10 @@ def main(argv): (labels, build_flags) = _get_build_flags(FLAGS.build[0]) build_desc = ",".join(labels) + with util.ProgressStep(f"Collecting configured targets for {build_desc}"): - cts = lib.analyze_build(BazelApi(), labels, build_flags) + cts = lib.analyze_build(BazelApi(bazel=FLAGS.bazel), labels, build_flags) + for analysis in FLAGS.analysis: analyses[analysis].exec(cts) diff --git a/tools/ctexplain/lib.py b/tools/ctexplain/lib.py index 172509d1c75f35..a97ded928df2ea 100644 --- a/tools/ctexplain/lib.py +++ b/tools/ctexplain/lib.py @@ -16,6 +16,7 @@ # Do not edit this line. Copybara replaces it with PY2 migration helper..third_party.bazel.tools.ctexplain.bazel_api as bazel_api from tools.ctexplain.types import ConfiguredTarget +import tools.ctexplain.bazel_api as bazel_api def analyze_build(bazel: bazel_api.BazelApi, labels: Tuple[str, ...],