diff --git a/.bazelci/postsubmit.yml b/.bazelci/postsubmit.yml index 6c277b82cf811d..5a7683db4ecca9 100644 --- a/.bazelci/postsubmit.yml +++ b/.bazelci/postsubmit.yml @@ -17,7 +17,7 @@ platforms: - "//src/tools/singlejar/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." ubuntu1604: shell_commands: - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# @@ -35,7 +35,7 @@ platforms: - "//src/tools/singlejar/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." ubuntu1804: shell_commands: - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# @@ -53,7 +53,7 @@ platforms: - "//src/tools/singlejar/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." ubuntu1804_nojava: build_flags: - "--javabase=@openjdk_linux_archive//:runtime" @@ -69,7 +69,7 @@ platforms: - "//src/test/..." - "//src/tools/singlejar/..." - "//third_party/ijar/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." # We can't run Android tests without an installed Android SDK / NDK. - "-//src/test/java/com/google/devtools/build/android/..." - "-//src/test/shell/bazel/android/..." @@ -127,7 +127,7 @@ platforms: - "//src/tools/singlejar/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." ubuntu1804_java10: shell_commands: - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# @@ -145,7 +145,7 @@ platforms: - "//src/tools/singlejar/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." ubuntu1804_java11: shell_commands: - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^# @@ -163,7 +163,7 @@ platforms: - "//src/tools/singlejar/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." # Some prebuilt jars doesn't run with Java 11 - "-//src/test/shell/bazel:external_integration_test" - "-//src/test/shell/bazel:maven_test" @@ -192,7 +192,7 @@ platforms: - "//src/tools/singlejar/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." - "-//src/test/shell/integration:minimal_jdk_test" windows: batch_commands: diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 01983e3b833cb2..2dcf73af0b936e 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -19,7 +19,7 @@ platforms: - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." # Disable Slow Tests - "-//src/test/shell/bazel:bazel_determinism_test" # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/4663 @@ -43,7 +43,7 @@ platforms: - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." # Disable Slow Tests - "-//src/test/shell/bazel:bazel_determinism_test" # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/4663 @@ -67,7 +67,7 @@ platforms: - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." # Disable Slow Tests - "-//src/test/shell/bazel:bazel_determinism_test" # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/4663 @@ -89,7 +89,7 @@ platforms: - "//src/tools/singlejar/..." - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." # Disable Slow Tests - "-//src/test/shell/bazel:bazel_determinism_test" # We can't run Android tests without an installed Android SDK / NDK. @@ -151,7 +151,7 @@ platforms: - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." # Disable Slow Tests - "-//src/test/shell/bazel:bazel_determinism_test" # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/4663 @@ -175,7 +175,7 @@ platforms: - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." # Disable Slow Tests - "-//src/test/shell/bazel:bazel_determinism_test" # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/4663 @@ -199,7 +199,7 @@ platforms: - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." # Disable Slow Tests - "-//src/test/shell/bazel:bazel_determinism_test" # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/4663 @@ -234,7 +234,7 @@ platforms: - "//src/tools/workspacelog/..." - "//third_party/ijar/..." - "//tools/android/..." - - "//tools/cmd_line_differ/..." + - "//tools/aquery_differ/..." # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/4663 - "-//src/test/shell/bazel/android:android_ndk_integration_test" # The below tests have been disabled because they are too slow on macOS. diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index 8221ef976c9c9c..b491f110268710 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -7,7 +7,7 @@ filegroup( "//src/test/shell/bazel/apple:srcs", "//src/test/shell/bazel/remote:srcs", "//src/test/shell/bazel/testdata:srcs", - "//tools/cmd_line_differ:srcs", + "//tools/aquery_differ:srcs", ], visibility = ["//src/test/shell:__pkg__"], ) diff --git a/tools/BUILD b/tools/BUILD index 03b14f3e68f8de..c13fab2960db40 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -8,11 +8,11 @@ filegroup( name = "srcs", srcs = glob(["**"]) + [ "//tools/android:srcs", + "//tools/aquery_differ:srcs", "//tools/bash:srcs", "//tools/buildstamp:srcs", "//tools/build_defs:srcs", "//tools/build_rules:srcs", - "//tools/cmd_line_differ:srcs", "//tools/coverage:srcs", "//tools/java:srcs", "//tools/jdk:srcs", @@ -71,9 +71,9 @@ test_suite( name = "all_windows_tests", tests = [ "//tools/android:all_windows_tests", + "//tools/aquery_differ:aquery_differ_test", "//tools/bash:all_windows_tests", "//tools/build_defs:all_windows_tests", - "//tools/cmd_line_differ:cmd_line_differ_test", "//tools/cpp/runfiles:all_windows_tests", "//tools/java:all_windows_tests", "//tools/jdk:all_windows_tests", diff --git a/tools/cmd_line_differ/BUILD b/tools/aquery_differ/BUILD similarity index 76% rename from tools/cmd_line_differ/BUILD rename to tools/aquery_differ/BUILD index 34bbffc1003e0a..fe3230da77be4d 100644 --- a/tools/cmd_line_differ/BUILD +++ b/tools/aquery_differ/BUILD @@ -21,20 +21,28 @@ filegroup( srcs = glob(["**"]), ) +py_library( + name = "aquery_differ_resolvers", + srcs = glob( + ["resolvers/*.py"], + ), +) + py_binary( - name = "cmd_line_differ", - srcs = ["cmd_line_differ.py"], + name = "aquery_differ", + srcs = ["aquery_differ.py"], deps = [ + ":aquery_differ_resolvers", "//src/main/protobuf:analysis_py_proto", "//third_party/py/abseil", ], ) py_test( - name = "cmd_line_differ_test", - srcs = ["cmd_line_differ_test.py"], + name = "aquery_differ_test", + srcs = ["aquery_differ_test.py"], deps = [ - ":cmd_line_differ", + ":aquery_differ", "//src/main/protobuf:analysis_py_proto", "//third_party/py/mock", ], diff --git a/tools/aquery_differ/aquery_differ.py b/tools/aquery_differ/aquery_differ.py new file mode 100644 index 00000000000000..64cdbb4aef9f58 --- /dev/null +++ b/tools/aquery_differ/aquery_differ.py @@ -0,0 +1,264 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +r"""Command line diffing tool that compares two bazel aquery invocations. + +This script compares the proto output of two bazel aquery invocations. For +each set of output files of an action, it compares the command lines that +generated the files. + +Example usage: +bazel aquery //path/to:target_one --output=textproto > \ + /path/to/output_one.textproto +bazel aquery //path/to:target_two --output=textproto > \ + /path/to/output_two.textproto + +From a bazel repo: +bazel run //tools/aquery_differ:aquery_differ -- \ +--before=/path/to/output_one.textproto \ +--after=/path/to/output_two.textproto +--input_type=textproto +--attrs=cmdline +--attrs=inputs +""" + +import difflib +import os +import sys +from absl import app +from absl import flags +from google.protobuf import text_format +from src.main.protobuf import analysis_pb2 +from tools.aquery_differ.resolvers.dep_set_resolver import DepSetResolver + +flags.DEFINE_string("before", None, "Aquery output before the change") +flags.DEFINE_string("after", None, "Aquery output after the change") +flags.DEFINE_enum( + "input_type", "proto", ["proto", "textproto"], + "The format of the aquery proto input. One of 'proto' and 'textproto.") +flags.DEFINE_multi_enum("attrs", ["cmdline"], ["inputs", "cmdline"], + "Attributes of the actions to be compared.") +flags.mark_flag_as_required("before") +flags.mark_flag_as_required("after") + +WHITE = "\033[37m%s\033[0m" +CYAN = "\033[36m%s\033[0m" +RED = "\033[31m%s\033[0m" +GREEN = "\033[32m%s\033[0m" + + +def _colorize(line): + """Add color to the input string.""" + if not sys.stdout.isatty(): + return line + + if line.startswith("+++") or line.startswith("---"): + return WHITE % line + + if line.startswith("@@"): + return CYAN % line + + if line.startswith("+"): + return GREEN % line + + if line.startswith("-"): + return RED % line + + return line + + +def _print_diff(output_files, before_val, after_val, attr, before_file, + after_file): + diff = "\n".join( + map(_colorize, [ + s.strip("\n") for s in difflib.unified_diff(before_val, after_val, + before_file, after_file) + ])) + print(("[%s]\n" + "Difference in the action that generates the following output(s):" + "\n\t%s\n%s\n") % (attr, "\n\t".join(output_files.split()), diff)) + + +def _map_artifact_id_to_path(artifacts): + return {artifact.id: artifact.exec_path for artifact in artifacts} + + +def _map_action_index_to_output_files(actions, artifacts): + """Constructs a map from action index to output files. + + Args: + actions: a list of actions from the action graph container + artifacts: a map {artifact_id: artifact path} + + Returns: + A map from action index (in action graph container) to a string of + concatenated output artifacts paths. + """ + action_index_to_output_files = {} + for i, action in enumerate(actions): + output_files = " ".join( + sorted([artifacts[output_id] for output_id in action.output_ids])) + action_index_to_output_files[i] = output_files + return action_index_to_output_files + + +# output files -> input artifacts +def _map_output_files_to_input_artifacts( + action_graph_container, artifact_id_to_path, action_index_to_output_files): + """Constructs a map from output files to input artifacts. + + Args: + action_graph_container: the full action graph container object + artifact_id_to_path: a map {artifact_id: artifact path} + action_index_to_output_files: a map from action index (in action graph + container) to a string of concatenated output artifacts paths. + + Returns: + A map from output files (string of concatenated output artifacts paths) to a + list of input artifacts. + """ + actions = action_graph_container.actions + dep_set_of_files = action_graph_container.dep_set_of_files + id_to_dep_set = {dep_set.id: dep_set for dep_set in dep_set_of_files} + dep_set_resolver = DepSetResolver(dep_set_of_files, artifact_id_to_path) + + output_files_to_input_artifacts = {} + for i, action in enumerate(actions): + input_artifacts = [] + + for dep_set_id in action.input_dep_set_ids: + input_artifacts.extend( + dep_set_resolver.resolve(id_to_dep_set[dep_set_id])) + + output_files_to_input_artifacts[action_index_to_output_files[i]] = list( + sorted(input_artifacts)) + + return output_files_to_input_artifacts + + +# output files -> command line +def _map_output_files_to_command_line(actions, action_index_to_output_files): + """Constructs a map from output files to command line. + + Args: + actions: a list of actions from the action graph container + action_index_to_output_files: a map from action index (in action graph + container) to a string of concatenated output artifacts paths. + + Returns: + A map from output files (string of concatenated output artifacts paths) + to the command line (a list of arguments). + """ + output_files_to_command_line = {} + for i, action in enumerate(actions): + output_files_to_command_line[ + action_index_to_output_files[i]] = action.arguments + return output_files_to_command_line + + +def _aquery_diff(before_proto, after_proto, attrs, before_file, after_file): + """Returns differences between command lines that generate same outputs.""" + found_difference = False + artifacts_before = _map_artifact_id_to_path(before_proto.artifacts) + artifacts_after = _map_artifact_id_to_path(after_proto.artifacts) + + action_to_output_files_before = _map_action_index_to_output_files( + before_proto.actions, artifacts_before) + action_to_output_files_after = _map_action_index_to_output_files( + after_proto.actions, artifacts_after) + + # There's a 1-to-1 mapping between action and outputs + output_files_before = set(action_to_output_files_before.values()) + output_files_after = set(action_to_output_files_after.values()) + + before_after_diff = output_files_before - output_files_after + after_before_diff = output_files_after - output_files_before + + if before_after_diff: + print(("Aquery output 'before' change contains an action that generates " + "the following outputs that aquery output 'after' change doesn't:" + "\n%s\n") % "\n".join(before_after_diff)) + found_difference = True + if after_before_diff: + print(("Aquery output 'after' change contains an action that generates " + "the following outputs that aquery output 'before' change doesn't:" + "\n%s\n") % "\n".join(after_before_diff)) + found_difference = True + + if "cmdline" in attrs: + output_to_command_line_before = _map_output_files_to_command_line( + before_proto.actions, action_to_output_files_before) + output_to_command_line_after = _map_output_files_to_command_line( + after_proto.actions, action_to_output_files_after) + for output_files in output_to_command_line_before: + arguments = output_to_command_line_before[output_files] + after_arguments = output_to_command_line_after.get(output_files, None) + if after_arguments and arguments != after_arguments: + _print_diff(output_files, arguments, after_arguments, "cmdline", + before_file, after_file) + found_difference = True + + if "inputs" in attrs: + output_to_input_files_before = _map_output_files_to_input_artifacts( + before_proto, artifacts_before, action_to_output_files_before) + output_to_input_files_after = _map_output_files_to_input_artifacts( + after_proto, artifacts_after, action_to_output_files_after) + for output_files in output_to_input_files_before: + before_inputs = output_to_input_files_before[output_files] + after_inputs = output_to_input_files_after.get(output_files, None) + if after_inputs and before_inputs != after_inputs: + _print_diff(output_files, before_inputs, after_inputs, "inputs", + before_file, after_file) + found_difference = True + + if not found_difference: + print("No difference") + + +def to_absolute_path(path): + path = os.path.expanduser(path) + if os.path.isabs(path): + return path + else: + if "BUILD_WORKING_DIRECTORY" in os.environ: + return os.path.join(os.environ["BUILD_WORKING_DIRECTORY"], path) + else: + return path + + +def main(unused_argv): + before_file = to_absolute_path(flags.FLAGS.before) + after_file = to_absolute_path(flags.FLAGS.after) + input_type = flags.FLAGS.input_type + attrs = flags.FLAGS.attrs + + before_proto = analysis_pb2.ActionGraphContainer() + after_proto = analysis_pb2.ActionGraphContainer() + if input_type == "proto": + with open(before_file, "rb") as f: + before_proto.ParseFromString(f.read()) + with open(after_file, "rb") as f: + after_proto.ParseFromString(f.read()) + else: + with open(before_file, "r") as f: + before_text = f.read() + text_format.Merge(before_text, before_proto) + with open(after_file, "r") as f: + after_text = f.read() + text_format.Merge(after_text, after_proto) + + _aquery_diff(before_proto, after_proto, attrs, before_file, after_file) + + +if __name__ == "__main__": + app.run(main) diff --git a/tools/cmd_line_differ/cmd_line_differ_test.py b/tools/aquery_differ/aquery_differ_test.py similarity index 62% rename from tools/cmd_line_differ/cmd_line_differ_test.py rename to tools/aquery_differ/aquery_differ_test.py index f5f8f2c9012947..4c51643c1fe424 100644 --- a/tools/cmd_line_differ/cmd_line_differ_test.py +++ b/tools/aquery_differ/aquery_differ_test.py @@ -14,7 +14,7 @@ import unittest from src.main.protobuf import analysis_pb2 -from tools.cmd_line_differ import cmd_line_differ +from tools.aquery_differ import aquery_differ from third_party.py import mock try: # Python 2 @@ -26,15 +26,34 @@ def make_aquery_output(actions, artifact_paths): action_graph = analysis_pb2.ActionGraphContainer() + for artifact_path in artifact_paths: next_id = len(action_graph.artifacts) artifact = action_graph.artifacts.add() artifact.id = str(next_id) artifact.exec_path = artifact_path + for next_action in actions: action = action_graph.actions.add() action.output_ids.extend(next_action["output_ids"]) action.arguments.extend(next_action["arguments"]) + + if "input_dep_set_ids" in next_action: + action.input_dep_set_ids.extend(next_action["input_dep_set_ids"]) + + return action_graph + + +def make_aquery_output_with_dep_set(actions, artifact_paths, dep_sets): + action_graph = make_aquery_output(actions, artifact_paths) + + for ds in dep_sets: + next_id = len(action_graph.dep_set_of_files) + dep_set = action_graph.dep_set_of_files.add() + dep_set.id = str(next_id) + dep_set.direct_artifact_ids.extend(ds["direct_artifact_ids"]) + dep_set.transitive_dep_set_ids.extend(ds["transitive_dep_set_ids"]) + return action_graph @@ -51,8 +70,10 @@ def test_no_difference(self): }], artifact_paths=["exec/path/zero", "exec/path/one", "exec/path/two"]) mock_stdout = StringIO() + attrs = ["cmdline"] with mock.patch("sys.stdout", mock_stdout): - cmd_line_differ._aquery_diff(action_graph, action_graph) + aquery_differ._aquery_diff(action_graph, action_graph, attrs, "before", + "after") self.assertEqual(mock_stdout.getvalue(), "No difference\n") def test_no_difference_different_output_files_order(self): @@ -74,8 +95,9 @@ def test_no_difference_different_output_files_order(self): artifact_paths=["exec/path/zero", "exec/path/one"]) mock_stdout = StringIO() + attrs = ["cmdline"] with mock.patch("sys.stdout", mock_stdout): - cmd_line_differ._aquery_diff(first, second) + aquery_differ._aquery_diff(first, second, attrs, "before", "after") self.assertEqual(mock_stdout.getvalue(), "No difference\n") def test_first_has_extra_output_files(self): @@ -102,12 +124,13 @@ def test_first_has_extra_output_files(self): artifact_paths=["exec/path/zero", "exec/path/one", "exec/path/two"], ) - expected_error = ("Aquery output before change contains an action " + expected_error = ("Aquery output 'before' change contains an action " "that generates the following outputs that aquery " - "output after change doesn't:\nexec/path/two\n\n") + "output 'after' change doesn't:\nexec/path/two\n\n") mock_stdout = StringIO() + attrs = ["cmdline"] with mock.patch("sys.stdout", mock_stdout): - cmd_line_differ._aquery_diff(first, second) + aquery_differ._aquery_diff(first, second, attrs, "before", "after") self.assertEqual(mock_stdout.getvalue(), expected_error) def test_second_has_extra_output_files(self): @@ -134,12 +157,13 @@ def test_second_has_extra_output_files(self): artifact_paths=["exec/path/zero", "exec/path/one", "exec/path/two"], ) - expected_error = ("Aquery output after change contains an action that" + expected_error = ("Aquery output 'after' change contains an action that" " generates the following outputs that aquery" - " output before change doesn't:\nexec/path/two\n\n") + " output 'before' change doesn't:\nexec/path/two\n\n") mock_stdout = StringIO() + attrs = ["cmdline"] with mock.patch("sys.stdout", mock_stdout): - cmd_line_differ._aquery_diff(first, second) + aquery_differ._aquery_diff(first, second, attrs, "before", "after") self.assertEqual(mock_stdout.getvalue(), expected_error) def test_different_command_lines(self): @@ -171,25 +195,64 @@ def test_different_command_lines(self): ) expected_error_one = "\n".join([ - "Difference in action that generates the following outputs:", - "exec/path/two", - "Aquery output before change has the following command line:", "-c", - "Aquery output after change has the following command line:", "-c", - "-d", "\n" + "Difference in the action that generates the following output(s):", + "\texec/path/two", "--- before", "+++ after", "@@ -1 +1,2 @@", " -c", + "+-d", "\n" ]) expected_error_two = "\n".join([ - "Difference in action that generates the following outputs:", - "exec/path/one", "exec/path/zero", - "Aquery output before change has the following command line:", "-a", - "-d", "Aquery output after change has the following command line:", - "-a", "-b", "\n" + "Difference in the action that generates the following output(s):", + "\texec/path/one", "\texec/path/zero", "--- before", "+++ after", + "@@ -1,2 +1,2 @@", " -a", "--d", "+-b", "\n" ]) + attrs = ["cmdline"] + mock_stdout = StringIO() with mock.patch("sys.stdout", mock_stdout): - cmd_line_differ._aquery_diff(first, second) + aquery_differ._aquery_diff(first, second, attrs, "before", "after") self.assertIn(expected_error_one, mock_stdout.getvalue()) self.assertIn(expected_error_two, mock_stdout.getvalue()) + def test_different_inputs(self): + first = make_aquery_output_with_dep_set( + actions=[{ + "arguments": [], + "output_ids": ["0", "1"], + "input_dep_set_ids": ["1"] + }], + artifact_paths=["exec/path/zero", "exec/path/one"], + dep_sets=[{ + "transitive_dep_set_ids": [], + "direct_artifact_ids": ["0"] + }, { + "transitive_dep_set_ids": ["0"], + "direct_artifact_ids": ["1"] + }]) + second = make_aquery_output_with_dep_set( + actions=[ + { + "arguments": [], + "output_ids": ["0", "1"], + "input_dep_set_ids": ["0"] + }, + ], + artifact_paths=["exec/path/zero", "exec/path/one"], + dep_sets=[{ + "transitive_dep_set_ids": [], + "direct_artifact_ids": ["0"] + }]) + + expected_error_one = "\n".join([ + "Difference in the action that generates the following output(s):", + "\texec/path/one", "\texec/path/zero", "--- before", "+++ after", + "@@ -1,2 +1 @@", "-exec/path/one", " exec/path/zero", "\n" + ]) + attrs = ["inputs"] + + mock_stdout = StringIO() + with mock.patch("sys.stdout", mock_stdout): + aquery_differ._aquery_diff(first, second, attrs, "before", "after") + self.assertIn(expected_error_one, mock_stdout.getvalue()) + if __name__ == "__main__": unittest.main() diff --git a/tools/aquery_differ/resolvers/dep_set_resolver.py b/tools/aquery_differ/resolvers/dep_set_resolver.py new file mode 100644 index 00000000000000..b5aa9710b7621b --- /dev/null +++ b/tools/aquery_differ/resolvers/dep_set_resolver.py @@ -0,0 +1,49 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Contains resolvers for different attributes of Action in aquery output.""" +import copy + + +class DepSetResolver(object): + """Utility class to resolve the dependency nested set.""" + + def __init__(self, dep_set_of_files, artifact_id_to_path): + self.dep_set_to_artifact_ids = {} + self.id_to_dep_set = {dep_set.id: dep_set for dep_set in dep_set_of_files} + self.artifact_id_to_path = artifact_id_to_path + + def resolve(self, dep_set): + """Given a dep set, return the flattened list of input artifact ids. + + Args: + dep_set: the dep set object to be resolved. + + Returns: + The flattened list of input artifact ids. + """ + if dep_set.id in self.dep_set_to_artifact_ids: + return self.dep_set_to_artifact_ids[dep_set.id] + + artifact_ids = copy.copy([ + self.artifact_id_to_path[artifact_id] + for artifact_id in dep_set.direct_artifact_ids + ]) + + for transitive_dep_set_id in dep_set.transitive_dep_set_ids: + artifact_ids.extend( + self.resolve(self.id_to_dep_set[transitive_dep_set_id])) + + self.dep_set_to_artifact_ids[dep_set.id] = artifact_ids + + return self.dep_set_to_artifact_ids[dep_set.id] diff --git a/tools/cmd_line_differ/cmd_line_differ.py b/tools/cmd_line_differ/cmd_line_differ.py deleted file mode 100644 index d05a4fedfa2e1e..00000000000000 --- a/tools/cmd_line_differ/cmd_line_differ.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2018 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -r"""Command line diffing tool that compares two bazel aquery invocations. - -This script compares the proto output of two bazel aquery invocations. For -each set of output files of an action, it compares the command lines that -generated the files. - -Example usage: -bazel aquery //path/to:target_one --output=textproto > \ - /path/to/output_one.textproto -bazel aquery //path/to:target_two --output=textproto > \ - /path/to/output_two.textproto - -From a bazel repo: -bazel run //tools/cmd_line_differ:cmd_line_differ -- \ ---before=/path/to/output_one.textproto \ ---after=/path/to/output_two.textproto ---input_type=textproto -""" - -import os -from absl import app -from absl import flags -from google.protobuf import text_format -from src.main.protobuf import analysis_pb2 - -flags.DEFINE_string("before", None, "Aquery output before the change") -flags.DEFINE_string("after", None, "Aquery output after the change") -flags.DEFINE_enum( - "input_type", "proto", ["proto", "textproto"], - "The format of the aquery proto input. One of 'proto' and 'textproto.") -flags.mark_flag_as_required("before") -flags.mark_flag_as_required("after") - - -def _map_artifact_id_to_path(artifacts): - return {artifact.id: artifact.exec_path for artifact in artifacts} - - -def _map_output_files_to_command_line(actions, artifacts): - output_files_to_command_line = {} - for action in actions: - output_files = " ".join( - sorted([artifacts[output_id] for output_id in action.output_ids])) - output_files_to_command_line[output_files] = action.arguments - return output_files_to_command_line - - -def _aquery_diff(before, after): - """Returns differences between command lines that generate same outputs.""" - # TODO(bazel-team): Currently we compare only command lines of actions that - # generate the same output files. Expand the differ to compare other values as - # well (e.g. mnemonic, inputs, execution tags...). - - found_difference = False - artifacts_before = _map_artifact_id_to_path(before.artifacts) - artifacts_after = _map_artifact_id_to_path(after.artifacts) - - output_to_command_line_before = _map_output_files_to_command_line( - before.actions, artifacts_before) - output_to_command_line_after = _map_output_files_to_command_line( - after.actions, artifacts_after) - - output_files_before = set(output_to_command_line_before.keys()) - output_files_after = set(output_to_command_line_after.keys()) - - before_after_diff = output_files_before - output_files_after - after_before_diff = output_files_after - output_files_before - - if before_after_diff: - print(("Aquery output before change contains an action that generates " - "the following outputs that aquery output after change doesn't:" - "\n%s\n") % "\n".join(before_after_diff)) - found_difference = True - if after_before_diff: - print(("Aquery output after change contains an action that generates " - "the following outputs that aquery output before change doesn't:" - "\n%s\n") % "\n".join(after_before_diff)) - found_difference = True - - for output_files in output_to_command_line_before: - arguments = output_to_command_line_before[output_files] - after_arguments = output_to_command_line_after.get(output_files, None) - if after_arguments and arguments != after_arguments: - print(("Difference in action that generates the following outputs:\n%s\n" - "Aquery output before change has the following command line:\n%s\n" - "Aquery output after change has the following command line:\n%s\n") - % ("\n".join(output_files.split()), "\n".join(arguments), - "\n".join(after_arguments))) - found_difference = True - - if not found_difference: - print("No difference") - - -def to_absolute_path(path): - path = os.path.expanduser(path) - if os.path.isabs(path): - return path - else: - if "BUILD_WORKING_DIRECTORY" in os.environ: - return os.path.join(os.environ["BUILD_WORKING_DIRECTORY"], path) - else: - return path - - -def main(unused_argv): - - before_file = to_absolute_path(flags.FLAGS.before) - after_file = to_absolute_path(flags.FLAGS.after) - input_type = flags.FLAGS.input_type - - before_proto = analysis_pb2.ActionGraphContainer() - after_proto = analysis_pb2.ActionGraphContainer() - if input_type == "proto": - with open(before_file, "rb") as f: - before_proto.ParseFromString(f.read()) - with open(after_file, "rb") as f: - after_proto.ParseFromString(f.read()) - else: - with open(before_file, "r") as f: - before_text = f.read() - text_format.Merge(before_text, before_proto) - with open(after_file, "r") as f: - after_text = f.read() - text_format.Merge(after_text, after_proto) - - _aquery_diff(before_proto, after_proto) - - -if __name__ == "__main__": - app.run(main)