diff --git a/MODULE.bazel b/MODULE.bazel index 5336c3654f50b7..49be7bd6d9c275 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -307,6 +307,30 @@ gvm.graalvm( ) use_repo(gvm, "graalvm_toolchains") +http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + +# DO NOT UPDATE the jq version, this is used to verify compatibility with old versions. +http_file( + name = "jq_linux_amd64", + executable = True, + integrity = "sha256-xrOn19PntwxvUbcGo7kL0BgzhGxU0yyjLwAn8AIm/20=", + urls = ["https://github.com/jqlang/jq/releases/download/jq-1.5/jq-linux64"], +) + +http_file( + name = "jq_macos_amd64", + executable = True, + integrity = "sha256-OG6SyYKlb+SFFGjXqTHfyilWDO4wag5mxqG9QGXT2sU=", + urls = ["https://github.com/jqlang/jq/releases/download/jq-1.5/jq-osx-amd64"], +) + +http_file( + name = "jq_windows_amd64", + executable = True, + integrity = "sha256-6+zYQLpH779mgihoF4zHIaFRBgk396xAbj0xvQFb3pQ=", + urls = ["https://github.com/jqlang/jq/releases/download/jq-1.5/jq-win64.exe"], +) + # ========================================= # Other Bazel testing dependencies # ========================================= diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 959e7f7128a0ae..db920441366925 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,6 +1,6 @@ { "lockFileVersion": 6, - "moduleFileHash": "ebb77f16d8f0a7ce4dfb1163ac6552073681a6fd506bd703e11bd435fa5badf5", + "moduleFileHash": "a7763ee017cb7133cd854e5217de5391d51c90faf7b9e2c412b37980b1952a32", "flags": { "cmdRegistries": [ "https://bcr.bazel.build/" @@ -333,7 +333,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 347, + "line": 371, "column": 22 } } @@ -522,13 +522,84 @@ "hasDevUseExtension": false, "hasNonDevUseExtension": true }, + { + "extensionBzlFile": "//:MODULE.bazel", + "extensionName": "_repo_rules", + "usingModule": "", + "location": { + "file": "@@//:MODULE.bazel", + "line": 0, + "column": 0 + }, + "imports": { + "jq_linux_amd64": "jq_linux_amd64", + "jq_macos_amd64": "jq_macos_amd64", + "jq_windows_amd64": "jq_windows_amd64" + }, + "devImports": [], + "tags": [ + { + "tagName": "@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributeValues": { + "executable": true, + "integrity": "sha256-xrOn19PntwxvUbcGo7kL0BgzhGxU0yyjLwAn8AIm/20=", + "urls": [ + "https://github.com/jqlang/jq/releases/download/jq-1.5/jq-linux64" + ], + "name": "jq_linux_amd64" + }, + "devDependency": false, + "location": { + "file": "@@//:MODULE.bazel", + "line": 313, + "column": 10 + } + }, + { + "tagName": "@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributeValues": { + "executable": true, + "integrity": "sha256-OG6SyYKlb+SFFGjXqTHfyilWDO4wag5mxqG9QGXT2sU=", + "urls": [ + "https://github.com/jqlang/jq/releases/download/jq-1.5/jq-osx-amd64" + ], + "name": "jq_macos_amd64" + }, + "devDependency": false, + "location": { + "file": "@@//:MODULE.bazel", + "line": 320, + "column": 10 + } + }, + { + "tagName": "@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributeValues": { + "executable": true, + "integrity": "sha256-6+zYQLpH779mgihoF4zHIaFRBgk396xAbj0xvQFb3pQ=", + "urls": [ + "https://github.com/jqlang/jq/releases/download/jq-1.5/jq-win64.exe" + ], + "name": "jq_windows_amd64" + }, + "devDependency": false, + "location": { + "file": "@@//:MODULE.bazel", + "line": 327, + "column": 10 + } + } + ], + "hasDevUseExtension": false, + "hasNonDevUseExtension": true + }, { "extensionBzlFile": "@io_bazel//:extensions.bzl", "extensionName": "bazel_test_deps", "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 314, + "line": 338, "column": 32 }, "imports": { @@ -547,7 +618,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 322, + "line": 346, "column": 31 }, "imports": { @@ -564,7 +635,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 325, + "line": 349, "column": 48 }, "imports": { @@ -581,7 +652,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 369, + "line": 393, "column": 35 }, "imports": { @@ -598,7 +669,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 372, + "line": 396, "column": 42 }, "imports": { @@ -2925,7 +2996,7 @@ "general": { "bzlTransitiveDigest": "fsj2Y0/OdubiefV/mXYFaPLbTvxWyuGu7Pp/xcVMWBE=", "recordedFileInputs": { - "@@//MODULE.bazel": "ebb77f16d8f0a7ce4dfb1163ac6552073681a6fd506bd703e11bd435fa5badf5", + "@@//MODULE.bazel": "a7763ee017cb7133cd854e5217de5391d51c90faf7b9e2c412b37980b1952a32", "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "828ca18c55ab0580454760a1a8b00d414057233329bd7a44c31e477cdaa56d81" }, "recordedDirentsInputs": {}, diff --git a/scripts/BUILD b/scripts/BUILD index 7e51e493b78c6e..6f872a4a8db817 100644 --- a/scripts/BUILD +++ b/scripts/BUILD @@ -40,6 +40,32 @@ sh_test( ], ) +filegroup( + name = "jq", + srcs = select({ + "@platforms//os:linux": ["@jq_linux_amd64//file"], + "@platforms//os:macos": ["@jq_macos_amd64//file"], + "@platforms//os:windows": ["@jq_windows_amd64//file"], + }), +) + +sh_test( + name = "bazel_lockfile_merge_test", + size = "small", + srcs = ["bazel_lockfile_merge_test.sh"], + data = [ + "bazel-lockfile-merge.jq", + "testenv.sh", + ":jq", + "//src/test/shell:bashunit", + "//src/test/tools/bzlmod:MODULE.bazel.lock", + "@bazel_tools//tools/bash/runfiles", + ], + env = { + "JQ_RLOCATIONPATH": "$(rlocationpath :jq)", + }, +) + filegroup( name = "srcs", srcs = glob(["**"]) + [ diff --git a/scripts/bazel-lockfile-merge.jq b/scripts/bazel-lockfile-merge.jq new file mode 100644 index 00000000000000..116af71dad99ae --- /dev/null +++ b/scripts/bazel-lockfile-merge.jq @@ -0,0 +1,54 @@ +# Merges an arbitrary number of MODULE.bazel.lock files. +# +# Input: an array of MODULE.bazel.lock JSON objects (as produced by `jq -s`). +# Output: a single MODULE.bazel.lock JSON object. +# +# This script assumes that all files are valid JSON and have a numeric +# "lockFileVersion" field. It will not fail on any such files, but only +# preserves information for files with a version of 10 or higher. +# +# The first file is considered to be the base when deciding which values to +# keep in case of conflicts. + +# Like unique, but preserves the order of the first occurrence of each element. +def stable_unique: + reduce .[] as $item ([]; if index($item) == null then . + [$item] else . end); + +# Given an array of objects, shallowly merges the result of applying f to each +# object into a single object, with a few special properties: +# 1. Values are uniquified before merging and then merged with last-wins +# semantics. Assuming that the first value is the base, this ensures that +# later occurrences of the base value do not override other values. For +# example, when this is called with B A1 A2 and A1 contains changes to a +# field but A2 does not (compared to B), the changes in A1 will be preserved. +# 2. Object keys on the top level are sorted lexicographically after merging, +# but are additionally split on ":". This ensures that module extension IDs, +# which start with labels, sort as strings in the same way as they due as +# structured objects in Bazel (that is, //python/extensions:python.bzl +# sorts before //python/extensions/private:internal_deps.bzl). +def shallow_merge(f): + map(f) | stable_unique | add | to_entries | sort_by(.key | split(":")) | from_entries; + +( + # Ignore all MODULE.bazel.lock files that do not have the maximum + # lockFileVersion. + (map(.lockFileVersion) | max) as $maxVersion + | map(select(.lockFileVersion == $maxVersion)) + | { + lockFileVersion: $maxVersion, + registryFileHashes: shallow_merge(.registryFileHashes), + selectedYankedVersions: shallow_merge(.selectedYankedVersions), + # Group extension results by extension ID across all lockfiles with + # shallowly merged factors map, then shallowly merge the results. + moduleExtensions: (map(.moduleExtensions | to_entries) + | flatten + | group_by(.key) + | shallow_merge({(.[0].key): shallow_merge(.value)})) + } +)? // + # We get here if the lockfiles with the highest lockFileVersion could not be + # processed, for example because all lockfiles have lockFileVersion < 10. + # In this case Bazel 7.2.0+ would ignore all lockfiles, so we might as well + # return the first lockfile for the proper "mismatched version" error + # message. + .[0] diff --git a/scripts/bazel_lockfile_merge_test.sh b/scripts/bazel_lockfile_merge_test.sh new file mode 100755 index 00000000000000..cbfdd638d15842 --- /dev/null +++ b/scripts/bazel_lockfile_merge_test.sh @@ -0,0 +1,273 @@ +#!/bin/bash +# +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# bash_completion_test.sh: tests of bash command completion. + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- + +: ${DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} +source ${DIR}/testenv.sh || { echo "testenv.sh not found!" >&2; exit 1; } + +JQ_SCRIPT_FILE="$(rlocation io_bazel/scripts/bazel-lockfile-merge.jq)" +JQ="$(rlocation $JQ_RLOCATIONPATH)" + +function do_merge() { + local base="$1" + local left="$2" + local right="$3" + + assert_not_contains "'" "$JQ_SCRIPT_FILE" + # Simulate the setup of a git merge driver, which can only be configured as a + # single command passed to sh and overwrites the "left" version. The check + # above verifies that wrapping the jq script in single quotes is sufficient to + # escape it here. + jq_script="$(cat "$JQ_SCRIPT_FILE")" + merge_cmd="\"$JQ\" -s '${jq_script}' -- $base $left $right > ${left}.jq_tmp && mv ${left}.jq_tmp ${left}" + sh -c "${merge_cmd}" || fail "merge failed" +} + +function test_synthetic_merge() { + cat > base <<'EOF' +{ + "lockFileVersion": 10, + "registryFileHashes": { + "https://example.org/modules/bar/0.9/MODULE.bazel": "1234", + "https://example.org/modules/foo/1.0/MODULE.bazel": "1234" + }, + "selectedYankedVersions": {}, + "moduleExtensions": { + "//:rbe_extensions.bzl%bazel_rbe_deps": { + "general": { + "bzlTransitiveDigest": "3Qxu4ylcYD3RTWLhk5k/59p/CwZ4tLdSgYnmBXYgAtc=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rbe_ubuntu2004": { + "bzlFile": "@@_main~bazel_test_deps~bazelci_rules//:rbe_repo.bzl", + "ruleClassName": "rbe_preconfig", + "attributes": { + "toolchain": "ubuntu2004" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "", + "bazelci_rules", + "_main~bazel_test_deps~bazelci_rules" + ] + ] + } + }, + "@@rules_python~//python/extensions:python.bzl%python": { + "general": { + "repo1": "old_args" + } + } + } +} +EOF + cat > left <<'EOF' +{ + "lockFileVersion": 10, + "registryFileHashes": { + "https://example.org/modules/bar/0.9/MODULE.bazel": "1234", + "https://example.org/modules/baz/2.0/MODULE.bazel": "1234", + "https://example.org/modules/foo/1.0/MODULE.bazel": "1234" + }, + "selectedYankedVersions": { + "bbb@1.0": "also dubious" + }, + "moduleExtensions": { + "@@rules_python~//python/extensions:python.bzl%python": { + "general": { + "repo1": "new_args" + } + }, + "@@rules_python~//python/extensions/private:internal_deps.bzl%internal_deps": { + "os:linux,arch:aarch64": { + "repo2": "aarch64_args" + } + } + } +} +EOF + cat > right <<'EOF' +{ + "lockFileVersion": 10, + "registryFileHashes": { + "https://example.org/modules/bar/0.9/MODULE.bazel": "1234", + "https://example.org/modules/bar/1.0/MODULE.bazel": "1234", + "https://example.org/modules/foo/1.0/MODULE.bazel": "1234" + }, + "selectedYankedVersions": { + "aaa@1.0": "dubious" + }, + "moduleExtensions": { + "//:rbe_extensions.bzl%bazel_rbe_deps": { + "general": { + "bzlTransitiveDigest": "changed", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rbe_ubuntu2004": { + "bzlFile": "@@_main~bazel_test_deps~bazelci_rules//:rbe_repo.bzl", + "ruleClassName": "rbe_preconfig", + "attributes": { + "toolchain": "ubuntu2004" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "", + "bazelci_rules", + "_main~bazel_test_deps~bazelci_rules" + ] + ] + } + }, + "@@rules_python~//python/extensions:python.bzl%python": { + "general": { + "repo1": "old_args" + } + }, + "@@rules_python~//python/extensions/private:internal_deps.bzl%internal_deps": { + "os:linux,arch:amd64": { + "repo2": "amd64_args" + } + } + } +} +EOF + cat > expected <<'EOF' +{ + "lockFileVersion": 10, + "registryFileHashes": { + "https://example.org/modules/bar/0.9/MODULE.bazel": "1234", + "https://example.org/modules/bar/1.0/MODULE.bazel": "1234", + "https://example.org/modules/baz/2.0/MODULE.bazel": "1234", + "https://example.org/modules/foo/1.0/MODULE.bazel": "1234" + }, + "selectedYankedVersions": { + "aaa@1.0": "dubious", + "bbb@1.0": "also dubious" + }, + "moduleExtensions": { + "//:rbe_extensions.bzl%bazel_rbe_deps": { + "general": { + "bzlTransitiveDigest": "changed", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rbe_ubuntu2004": { + "bzlFile": "@@_main~bazel_test_deps~bazelci_rules//:rbe_repo.bzl", + "ruleClassName": "rbe_preconfig", + "attributes": { + "toolchain": "ubuntu2004" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "", + "bazelci_rules", + "_main~bazel_test_deps~bazelci_rules" + ] + ] + } + }, + "@@rules_python~//python/extensions:python.bzl%python": { + "general": { + "repo1": "new_args" + } + }, + "@@rules_python~//python/extensions/private:internal_deps.bzl%internal_deps": { + "os:linux,arch:aarch64": { + "repo2": "aarch64_args" + }, + "os:linux,arch:amd64": { + "repo2": "amd64_args" + } + } + } +} +EOF + + do_merge base left right + diff -u expected left || fail "output differs" +} + +function test_complex_identity_merge() { + test_lockfile="$(rlocation io_bazel/src/test/tools/bzlmod/MODULE.bazel.lock)" + cp "$test_lockfile" base + cp "$test_lockfile" left + cp "$test_lockfile" right + + do_merge base left right + diff -u $test_lockfile left || fail "output differs" +} + +function test_merge_across_versions() { + test_lockfile="$(rlocation io_bazel/src/test/tools/bzlmod/MODULE.bazel.lock)" + cp "$test_lockfile" base + cp "$test_lockfile" left + cat > right <<'EOF' +{ + "lockFileVersion": 9, + "weirdField": {} +} +EOF + + do_merge base left right + diff -u $test_lockfile left || fail "output differs" +} + +function test_outdated_versions_only() { + cat > base <<'EOF' +{ + "lockFileVersion": 9, + "weirdField": {} +} +EOF + cat > left <<'EOF' +{ + "lockFileVersion": 8 +} +EOF + cat > right <<'EOF' +{ + "lockFileVersion": 7 +} +EOF + + do_merge base left right + diff -u base left || fail "output differs" +} + +run_suite "Tests of bash completion of 'blaze' command." diff --git a/site/en/external/lockfile.md b/site/en/external/lockfile.md index c8033569a3237e..6a4a600ca788c1 100644 --- a/site/en/external/lockfile.md +++ b/site/en/external/lockfile.md @@ -230,3 +230,49 @@ practices: By following these best practices, you can effectively utilize the lockfile feature in Bazel, leading to more efficient, reliable, and collaborative software development workflows. + +## Merge Conflicts {:#merge-conflicts} + +The lockfile format is designed to minimize merge conflicts, but they can still +happen. + +### Automatic Resolution {:#automatic-resolution} + +Bazel provides a custom +[git merge driver](https://git-scm.com/docs/gitattributes#_defining_a_custom_merge_driver) +to help resolve these conflicts automatically. + +Set up the driver by adding this line to a `.gitattributes` file in the root of +your git repository: + +```gitattributes +# A custom merge driver for the Bazel lockfile. +# https://bazel.build/external/lockfile#automatic-resolution +MODULE.bazel.lock merge=bazel-lockfile-merge +``` + +Then each developer who wants to use the driver has to register it once by +following these steps: + +1. Install [jq](https://jqlang.github.io/jq/download/) (1.5 or higher). +2. Run the following commands: + +```bash +jq_script=$(curl https://raw.githubusercontent.com/bazelbuild/bazel/master/scripts/bazel-lockfile-merge.jq) +printf '%s\n' "${jq_script}" | less # to optionally inspect the jq script +git config --global merge.bazel-lockfile-merge.name "Merge driver for the Bazel lockfile (MODULE.bazel.lock)" +git config --global merge.bazel-lockfile-merge.driver "jq -s '${jq_script}' -- %O %A %B > %A.jq_tmp && mv %A.jq_tmp %A" +``` + +### Manual Resolution {:#manual-resolution} + +Simple merge conflicts in the `registryFileHashes` and `selectedYankedVersions` +fields can be safely resolved by keeping all the entries from both sides of the +conflict. + +Other types of merge conflicts should not be resolved manually. Instead: + +1. Restore the previous state of the lockfile + via `git reset MODULE.bazel.lock && git checkout MODULE.bazel.lock`. +2. Resolve any conflicts in the `MODULE.bazel` file. +3. Run `bazel mod deps` to update the lockfile. diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index f34b03ba70c062..c89d830af424d4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -49,10 +49,13 @@ public class BazelLockFileFunction implements SkyFunction { private static final Pattern LOCKFILE_VERSION_PATTERN = Pattern.compile("\"lockFileVersion\":\\s*(\\d+)"); - private final Path rootDirectory; + private static final Pattern POSSIBLE_MERGE_CONFLICT_PATTERN = + Pattern.compile("<<<<<<<|=======|" + Pattern.quote("|||||||") + "|>>>>>>>"); private static final BazelLockFileValue EMPTY_LOCKFILE = BazelLockFileValue.builder().build(); + private final Path rootDirectory; + public BazelLockFileFunction(Path rootDirectory) { this.rootDirectory = rootDirectory; } @@ -71,13 +74,24 @@ public SkyValue compute(SkyKey skyKey, Environment env) try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "parse lockfile")) { return getLockfileValue(lockfilePath, LOCKFILE_MODE.get(env)); - } catch (IOException | JsonSyntaxException | NullPointerException e) { + } catch (IOException + | JsonSyntaxException + | NullPointerException + | IllegalArgumentException e) { + String actionSuffix; + if (POSSIBLE_MERGE_CONFLICT_PATTERN.matcher(e.getMessage()).find()) { + actionSuffix = + " This looks like a merge conflict. See" + + " https://bazel.build/external/lockfile#merge-conflicts for advice."; + } else { + actionSuffix = " Try deleting it and rerun the build."; + } throw new BazelLockfileFunctionException( ExternalDepsException.withMessage( Code.BAD_LOCKFILE, - "Failed to read and parse the MODULE.bazel.lock file with error: %s." - + " Try deleting it and rerun the build.", - e.getMessage()), + "Failed to read and parse the MODULE.bazel.lock file with error: %s.%s", + e.getMessage(), + actionSuffix), Transience.PERSISTENT); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java index 69622cef6ecba0..1e47b9b033cff5 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java @@ -196,13 +196,13 @@ public abstract static class RepoCacheFriendlyPath { public static RepoCacheFriendlyPath createInsideWorkspace( RepositoryName repoName, PathFragment path) { Preconditions.checkArgument( - !path.isAbsolute(), "the provided path should be relative to the repo root"); + !path.isAbsolute(), "the provided path should be relative to the repo root: %s", path); return new AutoValue_RepoRecordedInput_RepoCacheFriendlyPath(Optional.of(repoName), path); } public static RepoCacheFriendlyPath createOutsideWorkspace(PathFragment path) { Preconditions.checkArgument( - path.isAbsolute(), "the provided path should be absolute in the filesystem"); + path.isAbsolute(), "the provided path should be absolute in the filesystem: %s", path); return new AutoValue_RepoRecordedInput_RepoCacheFriendlyPath(Optional.empty(), path); }