diff --git a/third_party/py/frozendict/frozendict/__init__.py b/third_party/py/frozendict/frozendict/__init__.py index 399948a61b597b..cf51bcbbef40a8 100644 --- a/third_party/py/frozendict/frozendict/__init__.py +++ b/third_party/py/frozendict/frozendict/__init__.py @@ -9,11 +9,15 @@ except ImportError: # python < 2.7 OrderedDict = NotImplemented +try: + from collections import Mapping +except ImportError: + from collections.abc import Mapping iteritems = getattr(dict, 'iteritems', dict.items) # py2-3 compatibility -class frozendict(collections.Mapping): +class frozendict(Mapping): """ An immutable wrapper around dictionaries that implements the complete :py:class:`collections.Mapping` interface. It can be used as a drop-in replacement for dictionaries where immutability is desired. diff --git a/tools/ctexplain/analyses/summary.py b/tools/ctexplain/analyses/summary.py index 6669e6c5216445..e99a582016c13c 100644 --- a/tools/ctexplain/analyses/summary.py +++ b/tools/ctexplain/analyses/summary.py @@ -20,6 +20,8 @@ from tools.ctexplain.types import ConfiguredTarget # Do not edit this line. Copybara replaces it with PY2 migration helper..third_party.bazel.tools.ctexplain.util as util +import tools.ctexplain.util as util + @dataclass(frozen=True) class _Summary(): diff --git a/tools/ctexplain/bazel_api.py b/tools/ctexplain/bazel_api.py index 02a16174a6eb02..9d78986a56c664 100644 --- a/tools/ctexplain/bazel_api.py +++ b/tools/ctexplain/bazel_api.py @@ -15,8 +15,10 @@ There's no Python Bazel API so we invoke Bazel as a subprocess. """ +import functools import json import os +import re import subprocess from typing import Callable from typing import List @@ -28,8 +30,12 @@ from tools.ctexplain.types import HostConfiguration from tools.ctexplain.types import NullConfiguration +CQUERY_RESULT_LINE_REGEX = re.compile(r"^(.*?) \((\S+)\) \[([^[]*)\]$") +DEFAULT_BAZEL_BINARY = "bazel" -def run_bazel_in_client(args: List[str]) -> Tuple[int, List[str], List[str]]: + +def run_bazel_in_client(args: List[str], + bazel: str = DEFAULT_BAZEL_BINARY) -> Tuple[int, List[str], List[str]]: """Calls bazel within the current workspace. For production use. Tests use an alternative invoker that goes through test @@ -42,7 +48,7 @@ def run_bazel_in_client(args: List[str]) -> Tuple[int, List[str], List[str]]: Tuple of (return code, stdout, stderr) """ result = subprocess.run( - ["blaze"] + args, + [bazel] + args, cwd=os.getcwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -57,8 +63,12 @@ class BazelApi(): def __init__(self, run_bazel: Callable[[List[str]], Tuple[int, List[str], - List[str]]] = run_bazel_in_client): + List[str]]] = None, + bazel: str = DEFAULT_BAZEL_BINARY): + self.bazel = bazel self.run_bazel = run_bazel + if run_bazel is None: + self.run_bazel = functools.partial(run_bazel_in_client, bazel=self.bazel) def cquery(self, args: List[str]) -> Tuple[bool, str, Tuple[ConfiguredTarget, ...]]: @@ -144,20 +154,13 @@ def _parse_cquery_result_line(line: str) -> ConfiguredTarget: Returns: Corresponding ConfiguredTarget if the line matches else None. """ - tokens = line.split(maxsplit=2) - label = tokens[0] - if tokens[1][0] != "(" or tokens[1][-1] != ")": - raise ValueError(f"{tokens[1]} in {line} not surrounded by parentheses") - config_hash = tokens[1][1:-1] - if config_hash == "null": - fragments = () - else: - if tokens[2][0] != "[" or tokens[2][-1] != "]": - raise ValueError(f"{tokens[2]} in {line} not surrounded by [] brackets") - # The fragments list looks like '[Fragment1, Fragment2, ...]'. Split the - # whole line on ' [' to get just this list, then remove the final ']', then - # split again on ', ' to convert it to a structured tuple. - fragments = tuple(line.split(" [")[1][0:-1].split(", ")) + result = CQUERY_RESULT_LINE_REGEX.search(line) + if result is None: + raise ValueError(f"{repr(line)} does not match {repr(CQUERY_RESULT_LINE_REGEX.pattern)}") + label, config_hash, fragments_str = result.groups() + # The fragments list looks like 'Fragment1, Fragment2, ...'. Split on + # ', ' to convert it to a structured tuple. + fragments = tuple(fragments_str.split(", ")) return ConfiguredTarget( label=label, config=None, # Not yet available: we'll need `bazel config` to get this. diff --git a/tools/ctexplain/ctexplain.py b/tools/ctexplain/ctexplain.py index 65814799a35cb1..4798a943ff0594 100644 --- a/tools/ctexplain/ctexplain.py +++ b/tools/ctexplain/ctexplain.py @@ -43,11 +43,15 @@ from dataclasses import dataclass # Do not edit this line. Copybara replaces it with PY2 migration helper..third_party.bazel.tools.ctexplain.analyses.summary as summary -from tools.ctexplain.bazel_api import BazelApi +from tools.ctexplain.bazel_api import DEFAULT_BAZEL_BINARY, BazelApi # Do not edit this line. Copybara replaces it with PY2 migration helper..third_party.bazel.tools.ctexplain.lib as lib from tools.ctexplain.types import ConfiguredTarget # Do not edit this line. Copybara replaces it with PY2 migration helper..third_party.bazel.tools.ctexplain.util as util +import tools.ctexplain.lib as lib +import tools.ctexplain.util as util +import tools.ctexplain.analyses.summary as summary + FLAGS = flags.FLAGS @@ -112,6 +116,8 @@ def _render_analysis_help_text() -> str: lambda flag_value: all(name in analyses for name in flag_value), message=f'available analyses: {", ".join(analyses.keys())}') +flags.DEFINE_string("bazel", DEFAULT_BAZEL_BINARY, "Path to bazel binary") + flags.DEFINE_multi_string( "build", [], """command-line invocation of the build to analyze. For example: @@ -133,8 +139,8 @@ def _get_build_flags(cmdline: str) -> Tuple[Tuple[str, ...], Tuple[str, ...]]: Tuple of ((target labels to build), (build flags)) """ cmdlist = cmdline.split() - labels = [arg for arg in cmdlist if arg.startswith("//")] - build_flags = [arg for arg in cmdlist if not arg.startswith("//")] + labels = [arg for arg in cmdlist if arg.startswith("//") or arg.startswith("@")] + build_flags = [arg for arg in cmdlist if not arg.startswith("//") and not arg.startswith("@")] return (tuple(labels), tuple(build_flags)) @@ -148,8 +154,10 @@ def main(argv): (labels, build_flags) = _get_build_flags(FLAGS.build[0]) build_desc = ",".join(labels) + with util.ProgressStep(f"Collecting configured targets for {build_desc}"): - cts = lib.analyze_build(BazelApi(), labels, build_flags) + cts = lib.analyze_build(BazelApi(bazel=FLAGS.bazel), labels, build_flags) + for analysis in FLAGS.analysis: analyses[analysis].exec(cts) diff --git a/tools/ctexplain/lib.py b/tools/ctexplain/lib.py index 172509d1c75f35..a97ded928df2ea 100644 --- a/tools/ctexplain/lib.py +++ b/tools/ctexplain/lib.py @@ -16,6 +16,7 @@ # Do not edit this line. Copybara replaces it with PY2 migration helper..third_party.bazel.tools.ctexplain.bazel_api as bazel_api from tools.ctexplain.types import ConfiguredTarget +import tools.ctexplain.bazel_api as bazel_api def analyze_build(bazel: bazel_api.BazelApi, labels: Tuple[str, ...],