From 31872501d231cffdb5edc092862953f3b20803c5 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Wed, 5 Jun 2024 11:45:25 -0700 Subject: [PATCH] Add a git merge driver for `MODULE.bazel.lock` Adds a `jq` script to `scripts/` that merges any number of `MODULE.bazel.lock` files without using Bazel or reading the corresponding `MODULE.bazel` files. The lockfile docs now have a section explaining the steps needed to set up this script as a custom merger driver for Git, which means that merge conflicts in `MODULE.bazel.lock` files will always be resolved automatically. Note that resolution may emit lockfiles with redundant information that will be dropped by subsequent Bazel invocations. When Bazel encounters an error during lockfile parsing that could be caused by a merge conflict, it emits a different error message with a link to the docs. This required fixing the following kind of server crash when a conflict marker occurs inside a `recordedFileInputs` object: ``` FATAL: bazel crashed due to an internal error. Printing stack trace: java.lang.RuntimeException: Unrecoverable error while evaluating node 'com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileValue$$Lambda/0x000000f8011da998@314cd9ee' (requested by nodes 'RegistryKey{url=https://bcr.bazel.build/}') at com.google.devtools.build.skyframe.AbstractParallelEvaluator$Evaluate.run(AbstractParallelEvaluator.java:557) at com.google.devtools.build.lib.concurrent.AbstractQueueVisitor$WrappedRunnable.run(AbstractQueueVisitor.java:426) at java.base/java.util.concurrent.ForkJoinTask$AdaptedRunnableAction.exec(ForkJoinTask.java:1403) at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:387) at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1312) at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1843) at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1808) at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:188) Caused by: java.lang.IllegalArgumentException: the provided path should be absolute in the filesystem at com.google.common.base.Preconditions.checkArgument(Preconditions.java:143) at com.google.devtools.build.lib.rules.repository.RepoRecordedInput$RepoCacheFriendlyPath.createOutsideWorkspace(RepoRecordedInput.java:202) at com.google.devtools.build.lib.rules.repository.RepoRecordedInput$RepoCacheFriendlyPath.parse(RepoRecordedInput.java:222) at com.google.devtools.build.lib.rules.repository.RepoRecordedInput$File$1.parse(RepoRecordedInput.java:265) at com.google.devtools.build.lib.bazel.bzlmod.GsonTypeAdapterUtil$11.read(GsonTypeAdapterUtil.java:376) at com.google.devtools.build.lib.bazel.bzlmod.GsonTypeAdapterUtil$11.read(GsonTypeAdapterUtil.java:367) at com.google.gson.internal.bind.TypeAdapterRuntimeTypeWrapper.read(TypeAdapterRuntimeTypeWrapper.java:41) at com.google.gson.internal.bind.MapTypeAdapterFactory$Adapter.read(MapTypeAdapterFactory.java:186) at com.google.gson.internal.bind.MapTypeAdapterFactory$Adapter.read(MapTypeAdapterFactory.java:145) at com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory$1.read(DelegateTypeAdapterFactory.java:133) at com.google.devtools.build.lib.bazel.bzlmod.LockFileModuleExtension_GsonTypeAdapter.read(LockFileModuleExtension_GsonTypeAdapter.java:171) at com.google.devtools.build.lib.bazel.bzlmod.LockFileModuleExtension_GsonTypeAdapter.read(LockFileModuleExtension_GsonTypeAdapter.java:17) at com.google.gson.internal.bind.TypeAdapterRuntimeTypeWrapper.read(TypeAdapterRuntimeTypeWrapper.java:41) at com.google.gson.internal.bind.MapTypeAdapterFactory$Adapter.read(MapTypeAdapterFactory.java:187) at com.google.gson.internal.bind.MapTypeAdapterFactory$Adapter.read(MapTypeAdapterFactory.java:145) at com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory$1.read(DelegateTypeAdapterFactory.java:133) at com.google.gson.internal.bind.TypeAdapterRuntimeTypeWrapper.read(TypeAdapterRuntimeTypeWrapper.java:41) at com.google.gson.internal.bind.MapTypeAdapterFactory$Adapter.read(MapTypeAdapterFactory.java:187) at com.google.gson.internal.bind.MapTypeAdapterFactory$Adapter.read(MapTypeAdapterFactory.java:145) at com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory$1.read(DelegateTypeAdapterFactory.java:133) at com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileValue_GsonTypeAdapter.read(BazelLockFileValue_GsonTypeAdapter.java:129) at com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileValue_GsonTypeAdapter.read(BazelLockFileValue_GsonTypeAdapter.java:15) at com.google.gson.Gson.fromJson(Gson.java:991) at com.google.gson.Gson.fromJson(Gson.java:956) at com.google.gson.Gson.fromJson(Gson.java:905) at com.google.gson.Gson.fromJson(Gson.java:876) at com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction.getLockfileValue(BazelLockFileFunction.java:93) at com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction.compute(BazelLockFileFunction.java:73) at com.google.devtools.build.skyframe.AbstractParallelEvaluator$Evaluate.run(AbstractParallelEvaluator.java:468) ... 7 more ``` Alternatives considered: * Letting Bazel resolve the conflict would require building knowledge about particular version control systems and their conflict style into Bazel. It would also either require the user to resolve conflicts in `MODULE.bazel` first or deviate from the current behavior that the lockfile is not updated when any Bzlmod error is encountered. The jq script can be used as is by every VCS with merge driver support and resolves the conflict in `MODULE.bazel.lock` independently of `MODULE.bazel`. * Implementing the git merge driver as a `bazel mod` subcommand. This could be the source of intransparent slowdowns during regular git operations, which may even be triggered by other tools such as IDEs. The jq script is very fast. * Implementing the merger as a Go binary in buildtools would replace the ubiquitous jq tool with a special purpose binary while also not solving the problem that per-user action is required once to register a custom merge driver. Implements https://docs.google.com/document/d/1TjA7-M5njkI1F38IC0pm305S9EOmxcUwaCIvaSmansg/edit#heading=h.5mcn15i0e1ch RELNOTES: Git merge conflicts in `MODULE.bazel.lock` files can be resolved automatically. See https://bazel.build/external/lockfile#automatic-resolution for the required setup. Closes #22428. PiperOrigin-RevId: 640596606 Change-Id: I20659e3e53a7d8f2529f2ad5a3e7f258d7af026d --- MODULE.bazel | 24 ++ MODULE.bazel.lock | 89 +++++- scripts/BUILD | 26 ++ scripts/bazel-lockfile-merge.jq | 54 ++++ scripts/bazel_lockfile_merge_test.sh | 273 ++++++++++++++++++ site/en/external/lockfile.md | 46 +++ .../bazel/bzlmod/BazelLockFileFunction.java | 24 +- .../rules/repository/RepoRecordedInput.java | 4 +- 8 files changed, 524 insertions(+), 16 deletions(-) create mode 100644 scripts/bazel-lockfile-merge.jq create mode 100755 scripts/bazel_lockfile_merge_test.sh diff --git a/MODULE.bazel b/MODULE.bazel index 9dada1d8557a0c..b1be95aaa51160 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -324,6 +324,30 @@ gvm.graalvm( ) use_repo(gvm, "graalvm_toolchains") +http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + +# DO NOT UPDATE the jq version, this is used to verify compatibility with old versions. +http_file( + name = "jq_linux_amd64", + executable = True, + integrity = "sha256-xrOn19PntwxvUbcGo7kL0BgzhGxU0yyjLwAn8AIm/20=", + urls = ["https://github.com/jqlang/jq/releases/download/jq-1.5/jq-linux64"], +) + +http_file( + name = "jq_macos_amd64", + executable = True, + integrity = "sha256-OG6SyYKlb+SFFGjXqTHfyilWDO4wag5mxqG9QGXT2sU=", + urls = ["https://github.com/jqlang/jq/releases/download/jq-1.5/jq-osx-amd64"], +) + +http_file( + name = "jq_windows_amd64", + executable = True, + integrity = "sha256-6+zYQLpH779mgihoF4zHIaFRBgk396xAbj0xvQFb3pQ=", + urls = ["https://github.com/jqlang/jq/releases/download/jq-1.5/jq-win64.exe"], +) + # ========================================= # Other Bazel testing dependencies # ========================================= diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index f3148aee2167bf..977bc7333f63f1 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,6 +1,6 @@ { "lockFileVersion": 6, - "moduleFileHash": "c4124fbd3acadec360c37c532d1405f0c81f3c5388ef4079b525d2c3d9ce18e8", + "moduleFileHash": "3418b2507fd6534397a1371e7e9c73d58b538e23c981e0c5b3d258d9a0999ea4", "flags": { "cmdRegistries": [ "https://bcr.bazel.build/" @@ -333,7 +333,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 364, + "line": 388, "column": 22 } } @@ -522,13 +522,84 @@ "hasDevUseExtension": false, "hasNonDevUseExtension": true }, + { + "extensionBzlFile": "//:MODULE.bazel", + "extensionName": "_repo_rules", + "usingModule": "", + "location": { + "file": "@@//:MODULE.bazel", + "line": 0, + "column": 0 + }, + "imports": { + "jq_linux_amd64": "jq_linux_amd64", + "jq_macos_amd64": "jq_macos_amd64", + "jq_windows_amd64": "jq_windows_amd64" + }, + "devImports": [], + "tags": [ + { + "tagName": "@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributeValues": { + "executable": true, + "integrity": "sha256-xrOn19PntwxvUbcGo7kL0BgzhGxU0yyjLwAn8AIm/20=", + "urls": [ + "https://github.com/jqlang/jq/releases/download/jq-1.5/jq-linux64" + ], + "name": "jq_linux_amd64" + }, + "devDependency": false, + "location": { + "file": "@@//:MODULE.bazel", + "line": 330, + "column": 10 + } + }, + { + "tagName": "@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributeValues": { + "executable": true, + "integrity": "sha256-OG6SyYKlb+SFFGjXqTHfyilWDO4wag5mxqG9QGXT2sU=", + "urls": [ + "https://github.com/jqlang/jq/releases/download/jq-1.5/jq-osx-amd64" + ], + "name": "jq_macos_amd64" + }, + "devDependency": false, + "location": { + "file": "@@//:MODULE.bazel", + "line": 337, + "column": 10 + } + }, + { + "tagName": "@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributeValues": { + "executable": true, + "integrity": "sha256-6+zYQLpH779mgihoF4zHIaFRBgk396xAbj0xvQFb3pQ=", + "urls": [ + "https://github.com/jqlang/jq/releases/download/jq-1.5/jq-win64.exe" + ], + "name": "jq_windows_amd64" + }, + "devDependency": false, + "location": { + "file": "@@//:MODULE.bazel", + "line": 344, + "column": 10 + } + } + ], + "hasDevUseExtension": false, + "hasNonDevUseExtension": true + }, { "extensionBzlFile": "@io_bazel//:extensions.bzl", "extensionName": "bazel_test_deps", "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 331, + "line": 355, "column": 32 }, "imports": { @@ -547,7 +618,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 339, + "line": 363, "column": 31 }, "imports": { @@ -564,7 +635,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 342, + "line": 366, "column": 48 }, "imports": { @@ -581,7 +652,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 386, + "line": 410, "column": 35 }, "imports": { @@ -598,7 +669,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 389, + "line": 413, "column": 42 }, "imports": { @@ -616,7 +687,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 392, + "line": 416, "column": 45 }, "imports": { @@ -3019,7 +3090,7 @@ "general": { "bzlTransitiveDigest": "iHLxWy9Kdma4o6vQG1U0vO82d7jPg9gng3gCoY4Dn88=", "recordedFileInputs": { - "@@//MODULE.bazel": "c4124fbd3acadec360c37c532d1405f0c81f3c5388ef4079b525d2c3d9ce18e8", + "@@//MODULE.bazel": "3418b2507fd6534397a1371e7e9c73d58b538e23c981e0c5b3d258d9a0999ea4", "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "4a9cf4d1d48d36a3d6e24a13094b2c32aa20be5a495d4e5b33e43db973250182" }, "recordedDirentsInputs": {}, diff --git a/scripts/BUILD b/scripts/BUILD index 7e51e493b78c6e..6f872a4a8db817 100644 --- a/scripts/BUILD +++ b/scripts/BUILD @@ -40,6 +40,32 @@ sh_test( ], ) +filegroup( + name = "jq", + srcs = select({ + "@platforms//os:linux": ["@jq_linux_amd64//file"], + "@platforms//os:macos": ["@jq_macos_amd64//file"], + "@platforms//os:windows": ["@jq_windows_amd64//file"], + }), +) + +sh_test( + name = "bazel_lockfile_merge_test", + size = "small", + srcs = ["bazel_lockfile_merge_test.sh"], + data = [ + "bazel-lockfile-merge.jq", + "testenv.sh", + ":jq", + "//src/test/shell:bashunit", + "//src/test/tools/bzlmod:MODULE.bazel.lock", + "@bazel_tools//tools/bash/runfiles", + ], + env = { + "JQ_RLOCATIONPATH": "$(rlocationpath :jq)", + }, +) + filegroup( name = "srcs", srcs = glob(["**"]) + [ diff --git a/scripts/bazel-lockfile-merge.jq b/scripts/bazel-lockfile-merge.jq new file mode 100644 index 00000000000000..116af71dad99ae --- /dev/null +++ b/scripts/bazel-lockfile-merge.jq @@ -0,0 +1,54 @@ +# Merges an arbitrary number of MODULE.bazel.lock files. +# +# Input: an array of MODULE.bazel.lock JSON objects (as produced by `jq -s`). +# Output: a single MODULE.bazel.lock JSON object. +# +# This script assumes that all files are valid JSON and have a numeric +# "lockFileVersion" field. It will not fail on any such files, but only +# preserves information for files with a version of 10 or higher. +# +# The first file is considered to be the base when deciding which values to +# keep in case of conflicts. + +# Like unique, but preserves the order of the first occurrence of each element. +def stable_unique: + reduce .[] as $item ([]; if index($item) == null then . + [$item] else . end); + +# Given an array of objects, shallowly merges the result of applying f to each +# object into a single object, with a few special properties: +# 1. Values are uniquified before merging and then merged with last-wins +# semantics. Assuming that the first value is the base, this ensures that +# later occurrences of the base value do not override other values. For +# example, when this is called with B A1 A2 and A1 contains changes to a +# field but A2 does not (compared to B), the changes in A1 will be preserved. +# 2. Object keys on the top level are sorted lexicographically after merging, +# but are additionally split on ":". This ensures that module extension IDs, +# which start with labels, sort as strings in the same way as they due as +# structured objects in Bazel (that is, //python/extensions:python.bzl +# sorts before //python/extensions/private:internal_deps.bzl). +def shallow_merge(f): + map(f) | stable_unique | add | to_entries | sort_by(.key | split(":")) | from_entries; + +( + # Ignore all MODULE.bazel.lock files that do not have the maximum + # lockFileVersion. + (map(.lockFileVersion) | max) as $maxVersion + | map(select(.lockFileVersion == $maxVersion)) + | { + lockFileVersion: $maxVersion, + registryFileHashes: shallow_merge(.registryFileHashes), + selectedYankedVersions: shallow_merge(.selectedYankedVersions), + # Group extension results by extension ID across all lockfiles with + # shallowly merged factors map, then shallowly merge the results. + moduleExtensions: (map(.moduleExtensions | to_entries) + | flatten + | group_by(.key) + | shallow_merge({(.[0].key): shallow_merge(.value)})) + } +)? // + # We get here if the lockfiles with the highest lockFileVersion could not be + # processed, for example because all lockfiles have lockFileVersion < 10. + # In this case Bazel 7.2.0+ would ignore all lockfiles, so we might as well + # return the first lockfile for the proper "mismatched version" error + # message. + .[0] diff --git a/scripts/bazel_lockfile_merge_test.sh b/scripts/bazel_lockfile_merge_test.sh new file mode 100755 index 00000000000000..cbfdd638d15842 --- /dev/null +++ b/scripts/bazel_lockfile_merge_test.sh @@ -0,0 +1,273 @@ +#!/bin/bash +# +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# bash_completion_test.sh: tests of bash command completion. + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- + +: ${DIR:=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} +source ${DIR}/testenv.sh || { echo "testenv.sh not found!" >&2; exit 1; } + +JQ_SCRIPT_FILE="$(rlocation io_bazel/scripts/bazel-lockfile-merge.jq)" +JQ="$(rlocation $JQ_RLOCATIONPATH)" + +function do_merge() { + local base="$1" + local left="$2" + local right="$3" + + assert_not_contains "'" "$JQ_SCRIPT_FILE" + # Simulate the setup of a git merge driver, which can only be configured as a + # single command passed to sh and overwrites the "left" version. The check + # above verifies that wrapping the jq script in single quotes is sufficient to + # escape it here. + jq_script="$(cat "$JQ_SCRIPT_FILE")" + merge_cmd="\"$JQ\" -s '${jq_script}' -- $base $left $right > ${left}.jq_tmp && mv ${left}.jq_tmp ${left}" + sh -c "${merge_cmd}" || fail "merge failed" +} + +function test_synthetic_merge() { + cat > base <<'EOF' +{ + "lockFileVersion": 10, + "registryFileHashes": { + "https://example.org/modules/bar/0.9/MODULE.bazel": "1234", + "https://example.org/modules/foo/1.0/MODULE.bazel": "1234" + }, + "selectedYankedVersions": {}, + "moduleExtensions": { + "//:rbe_extensions.bzl%bazel_rbe_deps": { + "general": { + "bzlTransitiveDigest": "3Qxu4ylcYD3RTWLhk5k/59p/CwZ4tLdSgYnmBXYgAtc=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rbe_ubuntu2004": { + "bzlFile": "@@_main~bazel_test_deps~bazelci_rules//:rbe_repo.bzl", + "ruleClassName": "rbe_preconfig", + "attributes": { + "toolchain": "ubuntu2004" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "", + "bazelci_rules", + "_main~bazel_test_deps~bazelci_rules" + ] + ] + } + }, + "@@rules_python~//python/extensions:python.bzl%python": { + "general": { + "repo1": "old_args" + } + } + } +} +EOF + cat > left <<'EOF' +{ + "lockFileVersion": 10, + "registryFileHashes": { + "https://example.org/modules/bar/0.9/MODULE.bazel": "1234", + "https://example.org/modules/baz/2.0/MODULE.bazel": "1234", + "https://example.org/modules/foo/1.0/MODULE.bazel": "1234" + }, + "selectedYankedVersions": { + "bbb@1.0": "also dubious" + }, + "moduleExtensions": { + "@@rules_python~//python/extensions:python.bzl%python": { + "general": { + "repo1": "new_args" + } + }, + "@@rules_python~//python/extensions/private:internal_deps.bzl%internal_deps": { + "os:linux,arch:aarch64": { + "repo2": "aarch64_args" + } + } + } +} +EOF + cat > right <<'EOF' +{ + "lockFileVersion": 10, + "registryFileHashes": { + "https://example.org/modules/bar/0.9/MODULE.bazel": "1234", + "https://example.org/modules/bar/1.0/MODULE.bazel": "1234", + "https://example.org/modules/foo/1.0/MODULE.bazel": "1234" + }, + "selectedYankedVersions": { + "aaa@1.0": "dubious" + }, + "moduleExtensions": { + "//:rbe_extensions.bzl%bazel_rbe_deps": { + "general": { + "bzlTransitiveDigest": "changed", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rbe_ubuntu2004": { + "bzlFile": "@@_main~bazel_test_deps~bazelci_rules//:rbe_repo.bzl", + "ruleClassName": "rbe_preconfig", + "attributes": { + "toolchain": "ubuntu2004" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "", + "bazelci_rules", + "_main~bazel_test_deps~bazelci_rules" + ] + ] + } + }, + "@@rules_python~//python/extensions:python.bzl%python": { + "general": { + "repo1": "old_args" + } + }, + "@@rules_python~//python/extensions/private:internal_deps.bzl%internal_deps": { + "os:linux,arch:amd64": { + "repo2": "amd64_args" + } + } + } +} +EOF + cat > expected <<'EOF' +{ + "lockFileVersion": 10, + "registryFileHashes": { + "https://example.org/modules/bar/0.9/MODULE.bazel": "1234", + "https://example.org/modules/bar/1.0/MODULE.bazel": "1234", + "https://example.org/modules/baz/2.0/MODULE.bazel": "1234", + "https://example.org/modules/foo/1.0/MODULE.bazel": "1234" + }, + "selectedYankedVersions": { + "aaa@1.0": "dubious", + "bbb@1.0": "also dubious" + }, + "moduleExtensions": { + "//:rbe_extensions.bzl%bazel_rbe_deps": { + "general": { + "bzlTransitiveDigest": "changed", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rbe_ubuntu2004": { + "bzlFile": "@@_main~bazel_test_deps~bazelci_rules//:rbe_repo.bzl", + "ruleClassName": "rbe_preconfig", + "attributes": { + "toolchain": "ubuntu2004" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "", + "bazelci_rules", + "_main~bazel_test_deps~bazelci_rules" + ] + ] + } + }, + "@@rules_python~//python/extensions:python.bzl%python": { + "general": { + "repo1": "new_args" + } + }, + "@@rules_python~//python/extensions/private:internal_deps.bzl%internal_deps": { + "os:linux,arch:aarch64": { + "repo2": "aarch64_args" + }, + "os:linux,arch:amd64": { + "repo2": "amd64_args" + } + } + } +} +EOF + + do_merge base left right + diff -u expected left || fail "output differs" +} + +function test_complex_identity_merge() { + test_lockfile="$(rlocation io_bazel/src/test/tools/bzlmod/MODULE.bazel.lock)" + cp "$test_lockfile" base + cp "$test_lockfile" left + cp "$test_lockfile" right + + do_merge base left right + diff -u $test_lockfile left || fail "output differs" +} + +function test_merge_across_versions() { + test_lockfile="$(rlocation io_bazel/src/test/tools/bzlmod/MODULE.bazel.lock)" + cp "$test_lockfile" base + cp "$test_lockfile" left + cat > right <<'EOF' +{ + "lockFileVersion": 9, + "weirdField": {} +} +EOF + + do_merge base left right + diff -u $test_lockfile left || fail "output differs" +} + +function test_outdated_versions_only() { + cat > base <<'EOF' +{ + "lockFileVersion": 9, + "weirdField": {} +} +EOF + cat > left <<'EOF' +{ + "lockFileVersion": 8 +} +EOF + cat > right <<'EOF' +{ + "lockFileVersion": 7 +} +EOF + + do_merge base left right + diff -u base left || fail "output differs" +} + +run_suite "Tests of bash completion of 'blaze' command." diff --git a/site/en/external/lockfile.md b/site/en/external/lockfile.md index c8033569a3237e..6a4a600ca788c1 100644 --- a/site/en/external/lockfile.md +++ b/site/en/external/lockfile.md @@ -230,3 +230,49 @@ practices: By following these best practices, you can effectively utilize the lockfile feature in Bazel, leading to more efficient, reliable, and collaborative software development workflows. + +## Merge Conflicts {:#merge-conflicts} + +The lockfile format is designed to minimize merge conflicts, but they can still +happen. + +### Automatic Resolution {:#automatic-resolution} + +Bazel provides a custom +[git merge driver](https://git-scm.com/docs/gitattributes#_defining_a_custom_merge_driver) +to help resolve these conflicts automatically. + +Set up the driver by adding this line to a `.gitattributes` file in the root of +your git repository: + +```gitattributes +# A custom merge driver for the Bazel lockfile. +# https://bazel.build/external/lockfile#automatic-resolution +MODULE.bazel.lock merge=bazel-lockfile-merge +``` + +Then each developer who wants to use the driver has to register it once by +following these steps: + +1. Install [jq](https://jqlang.github.io/jq/download/) (1.5 or higher). +2. Run the following commands: + +```bash +jq_script=$(curl https://raw.githubusercontent.com/bazelbuild/bazel/master/scripts/bazel-lockfile-merge.jq) +printf '%s\n' "${jq_script}" | less # to optionally inspect the jq script +git config --global merge.bazel-lockfile-merge.name "Merge driver for the Bazel lockfile (MODULE.bazel.lock)" +git config --global merge.bazel-lockfile-merge.driver "jq -s '${jq_script}' -- %O %A %B > %A.jq_tmp && mv %A.jq_tmp %A" +``` + +### Manual Resolution {:#manual-resolution} + +Simple merge conflicts in the `registryFileHashes` and `selectedYankedVersions` +fields can be safely resolved by keeping all the entries from both sides of the +conflict. + +Other types of merge conflicts should not be resolved manually. Instead: + +1. Restore the previous state of the lockfile + via `git reset MODULE.bazel.lock && git checkout MODULE.bazel.lock`. +2. Resolve any conflicts in the `MODULE.bazel` file. +3. Run `bazel mod deps` to update the lockfile. diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index f34b03ba70c062..c89d830af424d4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -49,10 +49,13 @@ public class BazelLockFileFunction implements SkyFunction { private static final Pattern LOCKFILE_VERSION_PATTERN = Pattern.compile("\"lockFileVersion\":\\s*(\\d+)"); - private final Path rootDirectory; + private static final Pattern POSSIBLE_MERGE_CONFLICT_PATTERN = + Pattern.compile("<<<<<<<|=======|" + Pattern.quote("|||||||") + "|>>>>>>>"); private static final BazelLockFileValue EMPTY_LOCKFILE = BazelLockFileValue.builder().build(); + private final Path rootDirectory; + public BazelLockFileFunction(Path rootDirectory) { this.rootDirectory = rootDirectory; } @@ -71,13 +74,24 @@ public SkyValue compute(SkyKey skyKey, Environment env) try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "parse lockfile")) { return getLockfileValue(lockfilePath, LOCKFILE_MODE.get(env)); - } catch (IOException | JsonSyntaxException | NullPointerException e) { + } catch (IOException + | JsonSyntaxException + | NullPointerException + | IllegalArgumentException e) { + String actionSuffix; + if (POSSIBLE_MERGE_CONFLICT_PATTERN.matcher(e.getMessage()).find()) { + actionSuffix = + " This looks like a merge conflict. See" + + " https://bazel.build/external/lockfile#merge-conflicts for advice."; + } else { + actionSuffix = " Try deleting it and rerun the build."; + } throw new BazelLockfileFunctionException( ExternalDepsException.withMessage( Code.BAD_LOCKFILE, - "Failed to read and parse the MODULE.bazel.lock file with error: %s." - + " Try deleting it and rerun the build.", - e.getMessage()), + "Failed to read and parse the MODULE.bazel.lock file with error: %s.%s", + e.getMessage(), + actionSuffix), Transience.PERSISTENT); } } diff --git a/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java b/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java index 8e9d7fcf5c5c37..d8f0754385747f 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java +++ b/src/main/java/com/google/devtools/build/lib/rules/repository/RepoRecordedInput.java @@ -196,13 +196,13 @@ public abstract static class RepoCacheFriendlyPath { public static RepoCacheFriendlyPath createInsideWorkspace( RepositoryName repoName, PathFragment path) { Preconditions.checkArgument( - !path.isAbsolute(), "the provided path should be relative to the repo root"); + !path.isAbsolute(), "the provided path should be relative to the repo root: %s", path); return new AutoValue_RepoRecordedInput_RepoCacheFriendlyPath(Optional.of(repoName), path); } public static RepoCacheFriendlyPath createOutsideWorkspace(PathFragment path) { Preconditions.checkArgument( - path.isAbsolute(), "the provided path should be absolute in the filesystem"); + path.isAbsolute(), "the provided path should be absolute in the filesystem: %s", path); return new AutoValue_RepoRecordedInput_RepoCacheFriendlyPath(Optional.empty(), path); }