Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow a custom bazel binary name (or path) #191

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,9 @@ load("@hedron_compile_commands//:refresh_compile_commands.bzl", "refresh_compile
refresh_compile_commands(
name = "refresh_compile_commands",

# Change the name of the bazel executable (defaults to bazel)
# bazel_binary = "./bazel_wrapper"

# Specify the targets of interest.
# For example, specify a dict of targets and any flags required to build.
targets = {
Expand Down
45 changes: 26 additions & 19 deletions refresh.template.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,15 +96,15 @@ def _print_header_finding_warning_once():


@functools.lru_cache(maxsize=None)
def _get_bazel_version():
def _get_bazel_version(bazel_binary):
"""Gets the Bazel version as a tuple of (major, minor, patch).

The rolling release and the release candidate are treated as the LTS release.
E.g. both 7.0.0-pre.XXXXXXXX.X and 7.0.0rc1 are treated as 7.0.0.
If the version can't be determined, returns (0, 0, 0).
"""
bazel_version_process = subprocess.run(
['bazel', 'version'],
[bazel_binary, 'version'],
# MIN_PY=3.7: Replace PIPEs with capture_output.
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
Expand All @@ -127,10 +127,10 @@ def _get_bazel_version():


@functools.lru_cache(maxsize=None)
def _get_bazel_cached_action_keys():
def _get_bazel_cached_action_keys(bazel_binary):
"""Gets the set of actionKeys cached in bazel-out."""
action_cache_process = subprocess.run(
['bazel', 'dump', '--action_cache'],
[bazel_binary, 'dump', '--action_cache'],
# MIN_PY=3.7: Replace PIPEs with capture_output.
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
Expand Down Expand Up @@ -232,15 +232,15 @@ def _is_nvcc(path: str):
return os.path.basename(path).startswith('nvcc')


def _get_headers_gcc(compile_action, source_path: str, action_key: str):
def _get_headers_gcc(compile_action, source_path: str, action_key: str, bazel_binary: str):
"""Gets the headers used by a particular compile command that uses gcc arguments formatting (including clang.)

Relatively slow. Requires running the C preprocessor if we can't hit Bazel's cache.
"""
# Flags reference here: https://clang.llvm.org/docs/ClangCommandLineReference.html

# Check to see if Bazel has an (approximately) fresh cache of the included headers, and if so, use them to avoid a slow preprocessing step.
if action_key in _get_bazel_cached_action_keys(): # Safe because Bazel only holds one cached action key per path, and the key contains the path.
if action_key in _get_bazel_cached_action_keys(bazel_binary): # Safe because Bazel only holds one cached action key per path, and the key contains the path.
for i, arg in enumerate(compile_action.arguments):
if arg.startswith('-MF'):
if len(arg) > 3: # Either appended, like -MF<file>
Expand Down Expand Up @@ -517,7 +517,7 @@ def _file_is_in_main_workspace_and_not_external(file_str: str):
return True


def _get_headers(compile_action, source_path: str):
def _get_headers(compile_action, source_path: str, bazel_binary: str):
"""Gets the headers used by a particular compile command.

Relatively slow. Requires running the C preprocessor.
Expand Down Expand Up @@ -588,7 +588,7 @@ def _get_headers(compile_action, source_path: str):
if compile_action.arguments[0].endswith('cl.exe'): # cl.exe and also clang-cl.exe
headers, should_cache = _get_headers_msvc(compile_action, source_path)
else:
headers, should_cache = _get_headers_gcc(compile_action, source_path, compile_action.actionKey)
headers, should_cache = _get_headers_gcc(compile_action, source_path, compile_action.actionKey, bazel_binary)

# Cache for future use
if output_file and should_cache:
Expand All @@ -610,7 +610,7 @@ def _get_headers(compile_action, source_path: str):
_get_headers.has_logged = False


def _get_files(compile_action):
def _get_files(compile_action, bazel_binary):
"""Gets the ({source files}, {header files}) clangd should be told the command applies to."""

# Getting the source file is a little trickier than it might seem.
Expand Down Expand Up @@ -670,7 +670,7 @@ def _get_files(compile_action):
if os.path.splitext(source_file)[1] in _get_files.assembly_source_extensions:
return {source_file}, set()

header_files = _get_headers(compile_action, source_file)
header_files = _get_headers(compile_action, source_file, bazel_binary)

# Ambiguous .h headers need a language specified if they aren't C, or clangd sometimes makes mistakes
# Delete this and unused extension variables when clangd >= 16 is released, since their underlying issues are resolved at HEAD
Expand Down Expand Up @@ -1097,11 +1097,12 @@ def _nvcc_patch(compile_args: typing.List[str]) -> typing.List[str]:
}


def _get_cpp_command_for_files(compile_action):
def _get_cpp_command_for_files(compile_action, bazel_binary: str):
"""Reformat compile_action into a compile command clangd can understand.

Undo Bazel-isms and figures out which files clangd should apply the command to.
"""

# Condense aquery's environment variables into a dictionary, the format you might expect.
compile_action.environmentVariables = {pair.key: pair.value for pair in getattr(compile_action, 'environmentVariables', [])}
if 'PATH' not in compile_action.environmentVariables: # Bazel only adds if --incompatible_strict_action_env is passed--and otherwise inherits.
Expand All @@ -1113,15 +1114,15 @@ def _get_cpp_command_for_files(compile_action):
# Android and Linux and grailbio LLVM toolchains: Fine as is; no special patching needed.
compile_action.arguments = _all_platform_patch(compile_action.arguments)

source_files, header_files = _get_files(compile_action)
source_files, header_files = _get_files(compile_action, bazel_binary)

# Done after getting files since we may execute NVCC to get the files.
compile_action.arguments = _nvcc_patch(compile_action.arguments)

return source_files, header_files, compile_action.arguments


def _convert_compile_commands(aquery_output):
def _convert_compile_commands(aquery_output, bazel_binary: str):
"""Converts from Bazel's aquery format to de-Bazeled compile_commands.json entries.

Input: jsonproto output from aquery, pre-filtered to (Objective-)C(++) and CUDA compile actions for a given build.
Expand All @@ -1145,7 +1146,7 @@ def _convert_compile_commands(aquery_output):
with concurrent.futures.ThreadPoolExecutor(
max_workers=min(32, (os.cpu_count() or 1) + 4) # Backport. Default in MIN_PY=3.8. See "using very large resources implicitly on many-core machines" in https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
) as threadpool:
outputs = threadpool.map(_get_cpp_command_for_files, aquery_output.actions)
outputs = threadpool.map(lambda p: _get_cpp_command_for_files(*p), [(action, bazel_binary) for action in aquery_output.actions])

# Yield as compile_commands.json entries
header_files_already_written = set()
Expand All @@ -1171,7 +1172,7 @@ def _convert_compile_commands(aquery_output):
}


def _get_commands(target: str, flags: str):
def _get_commands(target: str, flags: str, bazel_binary: str):
"""Yields compile_commands.json entries for a given target and flags, gracefully tolerating errors."""
# Log clear completion messages
log_info(f">>> Analyzing commands used in {target}")
Expand Down Expand Up @@ -1200,7 +1201,7 @@ def _get_commands(target: str, flags: str):
# For efficiency, have bazel filter out external targets (and therefore actions) before they even get turned into actions or serialized and sent to us. Note: this is a different mechanism than is used for excluding just external headers.
target_statment = f"filter('^(//|@//)',{target_statment})"
aquery_args = [
'bazel',
bazel_binary,
'aquery',
# Aquery docs if you need em: https://docs.bazel.build/versions/master/aquery.html
# Aquery output proto reference: https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/analysis_v2.proto
Expand All @@ -1227,7 +1228,7 @@ def _get_commands(target: str, flags: str):
'--features=-layering_check',
]

if _get_bazel_version() >= (6, 1, 0):
if _get_bazel_version(bazel_binary) >= (6, 1, 0):
aquery_args += ['--host_features=-compiler_param_file', '--host_features=-layering_check']

aquery_args += additional_flags
Expand Down Expand Up @@ -1269,7 +1270,7 @@ def _get_commands(target: str, flags: str):
Continuing gracefully...""")
return

yield from _convert_compile_commands(parsed_aquery_output)
yield from _convert_compile_commands(parsed_aquery_output, bazel_binary)


# Log clear completion messages
Expand Down Expand Up @@ -1405,9 +1406,15 @@ def main():
# End: template filled by Bazel
]

bazel_binary = (
# Begin: template filled by Bazel
{bazel_binary}
# End: template filled by Bazel
)

compile_command_entries = []
for (target, flags) in target_flag_pairs:
compile_command_entries.extend(_get_commands(target, flags))
compile_command_entries.extend(_get_commands(target, flags, bazel_binary))

if not compile_command_entries:
log_error(""">>> Not (over)writing compile_commands.json, since no commands were extracted and an empty file is of no use.
Expand Down
5 changes: 4 additions & 1 deletion refresh_compile_commands.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def refresh_compile_commands(
targets = None,
exclude_headers = None,
exclude_external_sources = False,
bazel_binary = 'bazel',
**kwargs): # For the other common attributes. Tags, compatible_with, etc. https://docs.bazel.build/versions/main/be/common-definitions.html#common-attributes.
# Convert the various, acceptable target shorthands into the dictionary format
# In Python, `type(x) == y` is an antipattern, but [Starlark doesn't support inheritance](https://bazel.build/rules/language), so `isinstance` doesn't exist, and this is the correct way to switch on type.
Expand All @@ -89,7 +90,7 @@ def refresh_compile_commands(

# Generate the core, runnable python script from refresh.template.py
script_name = name + ".py"
_expand_template(name = script_name, labels_to_flags = targets, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
_expand_template(name = script_name, labels_to_flags = targets, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, bazel_binary = bazel_binary, **kwargs)

# Combine them so the wrapper calls the main script
native.py_binary(
Expand All @@ -115,12 +116,14 @@ def _expand_template_impl(ctx):
"{exclude_headers}": repr(ctx.attr.exclude_headers),
"{exclude_external_sources}": repr(ctx.attr.exclude_external_sources),
"{print_args_executable}": repr(ctx.executable._print_args_executable.path),
"{bazel_binary}": repr(ctx.attr.bazel_binary),
},
)
return DefaultInfo(files = depset([script]))

_expand_template = rule(
attrs = {
"bazel_binary": attr.string(default = 'bazel'),
"labels_to_flags": attr.string_dict(mandatory = True), # string keys instead of label_keyed because Bazel doesn't support parsing wildcard target patterns (..., *, :all) in BUILD attributes.
"exclude_external_sources": attr.bool(default = False),
"exclude_headers": attr.string(values = ["all", "external", ""]), # "" needed only for compatibility with Bazel < 3.6.0
Expand Down