diff --git a/.bazelci/postsubmit.yml b/.bazelci/postsubmit.yml index 5078fc2d638066..6f4d343363055d 100644 --- a/.bazelci/postsubmit.yml +++ b/.bazelci/postsubmit.yml @@ -331,6 +331,10 @@ tasks: - "-//src/java_tools/import_deps_checker/..." # We hit connection timeout error when downloading multiple URLs on RBE, see b/217865760 - "-//src/test/py/bazel:bazel_module_test" + - "-//src/test/py/bazel:bazel_lockfile_test" + - "-//src/test/py/bazel:bazel_overrides_test" + - "-//src/test/py/bazel:bazel_repo_mapping_test" + - "-//src/test/py/bazel:bazel_yanked_versions_test" - "-//src/test/shell/bazel:verify_workspace" include_json_profile: - build diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 8052bdd4d34f12..46cfb92eece6ee 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -321,6 +321,10 @@ tasks: - "-//src/java_tools/import_deps_checker/..." # We hit connection timeout error when downloading multiple URLs on RBE, see b/217865760 - "-//src/test/py/bazel:bazel_module_test" + - "-//src/test/py/bazel:bazel_lockfile_test" + - "-//src/test/py/bazel:bazel_overrides_test" + - "-//src/test/py/bazel:bazel_repo_mapping_test" + - "-//src/test/py/bazel:bazel_yanked_versions_test" - "-//src/test/shell/bazel:verify_workspace" kythe_ubuntu2004: shell_commands: diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 3a1250796735f0..26d05bbbef501f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -25,7 +25,9 @@ import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; +import com.google.devtools.build.lib.bazel.bzlmod.AttributeValues; import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; @@ -46,6 +48,7 @@ import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.RepositoryOverride; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; import com.google.devtools.build.lib.bazel.repository.downloader.DelegatingDownloader; @@ -139,6 +142,7 @@ public class BazelRepositoryModule extends BlazeModule { private final AtomicBoolean ignoreDevDeps = new AtomicBoolean(false); private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING; private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR; + private LockfileMode bazelLockfileMode = LockfileMode.OFF; private List allowedYankedVersions = ImmutableList.of(); private SingleExtensionEvalFunction singleExtensionEvalFunction; @@ -232,7 +236,8 @@ public void workspaceInit( public RepoSpec getRepoSpec(RepositoryName repoName) { return RepoSpec.builder() .setRuleClassName("local_config_platform") - .setAttributes(ImmutableMap.of("name", repoName.getName())) + .setAttributes( + AttributeValues.create(ImmutableMap.of("name", repoName.getName()))) .build(); } @@ -249,7 +254,10 @@ public ResolutionReason getResolutionReason() { .addSkyFunction( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules)) - .addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .addSkyFunction( + SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(directories.getWorkspace())) + .addSkyFunction( + SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace())) .addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction()) .addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) @@ -421,6 +429,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { ignoreDevDeps.set(repoOptions.ignoreDevDependency); checkDirectDepsMode = repoOptions.checkDirectDependencies; bazelCompatibilityMode = repoOptions.bazelCompatibilityMode; + bazelLockfileMode = repoOptions.lockfileMode; allowedYankedVersions = repoOptions.allowedYankedVersions; if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) { @@ -512,6 +521,7 @@ public ImmutableList getPrecomputedValues() { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, checkDirectDepsMode), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, bazelCompatibilityMode), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode), PrecomputedValue.injected( BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, allowedYankedVersions)); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java index 15c2eb23976391..3b384a709ff8bd 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java @@ -89,7 +89,7 @@ public RepoSpec build() { return RepoSpec.builder() .setBzlFile("@bazel_tools//tools/build_defs/repo:http.bzl") .setRuleClassName("http_archive") - .setAttributes(attrBuilder.buildOrThrow()) + .setAttributes(AttributeValues.create(attrBuilder.buildOrThrow())) .build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValues.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValues.java new file mode 100644 index 00000000000000..d73e4ac443bff9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValues.java @@ -0,0 +1,58 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.Maps; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; +import java.util.List; +import java.util.Map; +import net.starlark.java.eval.Dict; +import net.starlark.java.eval.Starlark; + +/** Wraps a dictionary of attribute names and values. Always uses a dict to represent them */ +@AutoValue +@GenerateTypeAdapter +public abstract class AttributeValues { + + public static AttributeValues create(Dict attribs) { + return new AutoValue_AttributeValues(attribs); + } + + public static AttributeValues create(Map attribs) { + return new AutoValue_AttributeValues( + Dict.immutableCopyOf(Maps.transformValues(attribs, AttributeValues::valueToStarlark))); + } + + public abstract Dict attributes(); + + // TODO(salmasamy) this is a copy of Attribute::valueToStarlark, Maybe think of a better place? + private static Object valueToStarlark(Object x) { + // Is x a non-empty string_list_dict? + if (x instanceof Map) { + Map map = (Map) x; + if (!map.isEmpty() && map.values().iterator().next() instanceof List) { + Dict.Builder dict = Dict.builder(); + for (Map.Entry e : map.entrySet()) { + dict.put(e.getKey(), Starlark.fromJava(e.getValue(), null)); + } + return dict.buildImmutable(); + } + } + // For all other attribute values, shallow conversion is safe. + return Starlark.fromJava(x, null); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValuesAdapter.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValuesAdapter.java new file mode 100644 index 00000000000000..c178ed60717e01 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValuesAdapter.java @@ -0,0 +1,156 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.devtools.build.lib.cmdline.Label; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import net.starlark.java.eval.Dict; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Mutability; +import net.starlark.java.eval.Starlark; +import net.starlark.java.eval.StarlarkInt; +import net.starlark.java.eval.StarlarkList; + +/** Helps serialize/deserialize {@link AttributeValues}, which contains Starlark values. */ +public class AttributeValuesAdapter extends TypeAdapter { + + @Override + public void write(JsonWriter out, AttributeValues attributeValues) throws IOException { + JsonObject jsonObject = new JsonObject(); + for (Map.Entry entry : attributeValues.attributes().entrySet()) { + jsonObject.add(entry.getKey(), serializeObject(entry.getValue())); + } + out.jsonValue(jsonObject.toString()); + } + + @Override + public AttributeValues read(JsonReader in) throws IOException { + JsonObject jsonObject = JsonParser.parseReader(in).getAsJsonObject(); + Dict.Builder dict = Dict.builder(); + for (Map.Entry entry : jsonObject.entrySet()) { + dict.put(entry.getKey(), deserializeObject(entry.getValue())); + } + return AttributeValues.create(dict.buildImmutable()); + } + + /** + * Starlark Object Types Bool Integer String Label List (Int, label, string) Dict (String,list) & + * (Label, String) + */ + private JsonElement serializeObject(Object obj) { + if (obj.equals(Starlark.NONE)) { + return JsonNull.INSTANCE; + } else if (obj instanceof Boolean) { + return new JsonPrimitive((Boolean) obj); + } else if (obj instanceof StarlarkInt) { + try { + return new JsonPrimitive(((StarlarkInt) obj).toInt("serialization into the lockfile")); + } catch (EvalException e) { + throw new IllegalArgumentException("Unable to parse StarlarkInt to Integer: " + e); + } + } else if (obj instanceof String || obj instanceof Label) { + return new JsonPrimitive(serializeObjToString(obj)); + } else if (obj instanceof Dict) { + JsonObject jsonObject = new JsonObject(); + for (Map.Entry entry : ((Dict) obj).entrySet()) { + jsonObject.add(serializeObjToString(entry.getKey()), serializeObject(entry.getValue())); + } + return jsonObject; + } else if (obj instanceof StarlarkList) { + JsonArray jsonArray = new JsonArray(); + for (Object item : (StarlarkList) obj) { + jsonArray.add(serializeObject(item)); + } + return jsonArray; + } else { + throw new IllegalArgumentException("Unsupported type: " + obj.getClass()); + } + } + + private Object deserializeObject(JsonElement json) { + if (json == null || json.isJsonNull()) { + return Starlark.NONE; + } else if (json.isJsonPrimitive()) { + JsonPrimitive jsonPrimitive = json.getAsJsonPrimitive(); + if (jsonPrimitive.isBoolean()) { + return jsonPrimitive.getAsBoolean(); + } else if (jsonPrimitive.isNumber()) { + return StarlarkInt.of(jsonPrimitive.getAsInt()); + } else if (jsonPrimitive.isString()) { + return deserializeStringToObject(jsonPrimitive.getAsString()); + } else { + throw new IllegalArgumentException("Unsupported JSON primitive: " + jsonPrimitive); + } + } else if (json.isJsonObject()) { + JsonObject jsonObject = json.getAsJsonObject(); + Dict.Builder dict = Dict.builder(); + for (Map.Entry entry : jsonObject.entrySet()) { + dict.put(deserializeStringToObject(entry.getKey()), deserializeObject(entry.getValue())); + } + return dict.buildImmutable(); + } else if (json.isJsonArray()) { + JsonArray jsonArray = json.getAsJsonArray(); + List list = new ArrayList<>(); + for (JsonElement item : jsonArray) { + list.add(deserializeObject(item)); + } + return StarlarkList.copyOf(Mutability.IMMUTABLE, list); + } else { + throw new IllegalArgumentException("Unsupported JSON element: " + json); + } + } + + /** + * Serializes an object (Label or String) to String A label is converted to a String as it is. A + * String is being modified with a delimiter to be easily differentiated from the label when + * deserializing. + * + * @param obj String or Label + * @return serialized object + */ + private String serializeObjToString(Object obj) { + if (obj instanceof Label) { + return ((Label) obj).getUnambiguousCanonicalForm(); + } + return "--" + obj; + } + + /** + * Deserializes a string to either a label or a String depending on the prefix. A string will have + * a delimiter at the start, else it is converted to a label. + * + * @param value String to be deserialized + * @return Object of type String of Label + */ + private Object deserializeStringToObject(String value) { + if (value.startsWith("--")) { + return value.substring(2); + } + return Label.parseCanonicalUnchecked(value); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index dc0ff89bbbb598..bc24b3319356fd 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -14,6 +14,7 @@ java_library( name = "common", srcs = [ "ArchiveRepoSpecBuilder.java", + "AttributeValues.java", "ModuleKey.java", "RepoSpec.java", "Version.java", @@ -22,6 +23,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/net/starlark/java/eval", "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", ], @@ -49,6 +51,7 @@ java_library( "TagClass.java", ], deps = [ + ":common", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", @@ -56,8 +59,8 @@ java_library( "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", - "//third_party:jsr305", ], ) @@ -87,7 +90,9 @@ java_library( "AbridgedModule.java", "ArchiveOverride.java", "BazelDepGraphValue.java", + "BazelLockFileValue.java", "BazelModuleResolutionValue.java", + "BzlmodFlagsAndEnvVars.java", "GitOverride.java", "LocalPathOverride.java", "Module.java", @@ -119,6 +124,7 @@ java_library( "//src/main/java/net/starlark/java/eval", "//src/main/java/net/starlark/java/syntax", "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", ], @@ -127,9 +133,14 @@ java_library( java_library( name = "resolution_impl", srcs = [ + "AttributeValuesAdapter.java", "BazelDepGraphFunction.java", + "BazelLockFileFunction.java", "BazelModuleResolutionFunction.java", + "BzlmodFlagsAndEnvVars.java", + "DelegateTypeAdapterFactory.java", "Discovery.java", + "GsonTypeAdapterUtil.java", "ModuleExtensionContext.java", "ModuleFileFunction.java", "ModuleFileGlobals.java", @@ -157,6 +168,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", + "//src/main/java/com/google/devtools/build/lib/packages/semantics", "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_directory_value", "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function", "//src/main/java/com/google/devtools/build/lib/skyframe:bzl_load_value", @@ -164,6 +176,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/skyframe:client_environment_value", "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster", + "//src/main/java/com/google/devtools/build/lib/util", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/skyframe", @@ -174,6 +187,7 @@ java_library( "//src/main/java/net/starlark/java/syntax", "//src/main/protobuf:failure_details_java_proto", "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", ], diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index dcc6b8e94d9034..14bfbdafc730f0 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -15,25 +15,36 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableTable; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction.BazelModuleResolutionFunctionException; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.packages.LabelConverter; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import javax.annotation.Nullable; /** @@ -43,24 +54,154 @@ */ public class BazelDepGraphFunction implements SkyFunction { + private final Path rootDirectory; + + public BazelDepGraphFunction(Path rootDirectory) { + this.rootDirectory = rootDirectory; + } + @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException { - - BazelModuleResolutionValue selectionResult = - (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); - if (env.valuesMissing()) { + RootModuleFileValue root = + (RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE); + if (root == null) { return null; } + LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); + + ImmutableMap localOverrideHashes = null; + ImmutableMap depGraph = null; + BzlmodFlagsAndEnvVars flags = null; + + // If the module has not changed (has the same contents and flags as the lockfile), + // read the dependency graph from the lock file, else run resolution and update lockfile + if (!lockfileMode.equals(LockfileMode.OFF)) { + BazelLockFileValue lockFile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); + if (lockFile == null) { + return null; + } + flags = getFlagsAndEnvVars(env); + if (flags == null) { // unable to read environment variables + return null; + } + localOverrideHashes = getLocalOverridesHashes(root.getOverrides(), env); + if (localOverrideHashes == null) { // trying to read override module + return null; + } + + if (root.getModuleFileHash().equals(lockFile.getModuleFileHash()) + && flags.equals(lockFile.getFlags()) + && localOverrideHashes.equals(lockFile.getLocalOverrideHashes())) { + depGraph = lockFile.getModuleDepGraph(); + } else if (lockfileMode.equals(LockfileMode.ERROR)) { + List diffLockfile = + lockFile.getDiffLockfile(root.getModuleFileHash(), localOverrideHashes, flags); + throw new BazelDepGraphFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "Lock file is no longer up-to-date because: %s", + String.join(", ", diffLockfile)), + Transience.PERSISTENT); + } + } + + if (depGraph == null) { + BazelModuleResolutionValue selectionResult = + (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); + if (env.valuesMissing()) { + return null; + } + depGraph = selectionResult.getResolvedDepGraph(); + if (lockfileMode.equals(LockfileMode.UPDATE)) { + BazelLockFileFunction.updateLockedModule( + rootDirectory, root.getModuleFileHash(), flags, localOverrideHashes, depGraph); + } + } - ImmutableMap depGraph = selectionResult.getResolvedDepGraph(); ImmutableMap canonicalRepoNameLookup = depGraph.keySet().stream() .collect(toImmutableMap(ModuleKey::getCanonicalRepoName, key -> key)); - // For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can - // group all usages by the label + name (the ModuleExtensionId). + ImmutableTable extensionUsagesById = + getExtensionUsagesById(depGraph); + + ImmutableBiMap extensionUniqueNames = + calculateUniqueNameForUsedExtensionId(extensionUsagesById); + + return BazelDepGraphValue.create( + depGraph, + canonicalRepoNameLookup, + depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()), + extensionUsagesById, + extensionUniqueNames.inverse()); + } + + @Nullable + @VisibleForTesting + static ImmutableMap getLocalOverridesHashes( + Map overrides, Environment env) throws InterruptedException { + ImmutableMap.Builder localOverrideHashes = new ImmutableMap.Builder<>(); + for (Entry entry : overrides.entrySet()) { + if (entry.getValue() instanceof LocalPathOverride) { + ModuleFileValue moduleValue = + (ModuleFileValue) + env.getValue( + ModuleFileValue.key( + ModuleKey.create(entry.getKey(), Version.EMPTY), entry.getValue())); + if (moduleValue == null) { + return null; + } + localOverrideHashes.put(entry.getKey(), moduleValue.getModuleFileHash()); + } + } + return localOverrideHashes.buildOrThrow(); + } + + @VisibleForTesting + @Nullable + static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws InterruptedException { + ClientEnvironmentValue allowedYankedVersionsFromEnv = + (ClientEnvironmentValue) + env.getValue( + ClientEnvironmentFunction.key( + BazelModuleResolutionFunction.BZLMOD_ALLOWED_YANKED_VERSIONS_ENV)); + if (allowedYankedVersionsFromEnv == null) { + return null; + } + + ImmutableList registries = ImmutableList.copyOf(ModuleFileFunction.REGISTRIES.get(env)); + ImmutableMap moduleOverrides = + ModuleFileFunction.MODULE_OVERRIDES.get(env).entrySet().stream() + .collect( + toImmutableMap(e -> e.getKey(), e -> ((LocalPathOverride) e.getValue()).getPath())); + + ImmutableList yankedVersions = + ImmutableList.copyOf(BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.get(env)); + Boolean ignoreDevDeps = ModuleFileFunction.IGNORE_DEV_DEPS.get(env); + String compatabilityMode = + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.get(env).name(); + String directDepsMode = BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.get(env).name(); + + String envYanked = allowedYankedVersionsFromEnv.getValue(); + + return BzlmodFlagsAndEnvVars.create( + registries, + moduleOverrides, + yankedVersions, + nullToEmpty(envYanked), + ignoreDevDeps, + directDepsMode, + compatabilityMode); + } + + /** + * For each extension usage, we resolve (i.e. canonicalize) its bzl file label. Then we can group + * all usages by the label + name (the ModuleExtensionId). + */ + private ImmutableTable getExtensionUsagesById( + ImmutableMap depGraph) throws BazelDepGraphFunctionException { ImmutableTable.Builder extensionUsagesTableBuilder = ImmutableTable.builder(); for (Module module : depGraph.values()) { @@ -75,7 +216,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) ModuleExtensionId.create( labelConverter.convert(usage.getExtensionBzlFile()), usage.getExtensionName()); } catch (LabelSyntaxException e) { - throw new BazelModuleResolutionFunctionException( + throw new BazelDepGraphFunctionException( ExternalDepsException.withCauseAndMessage( Code.BAD_MODULE, e, @@ -84,7 +225,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) Transience.PERSISTENT); } if (!moduleExtensionId.getBzlFileLabel().getRepository().isVisible()) { - throw new BazelModuleResolutionFunctionException( + throw new BazelDepGraphFunctionException( ExternalDepsException.withMessage( Code.BAD_MODULE, "invalid label for module extension found at %s: no repo visible as '@%s' here", @@ -95,21 +236,18 @@ public SkyValue compute(SkyKey skyKey, Environment env) extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage); } } - ImmutableTable extensionUsagesById = - extensionUsagesTableBuilder.buildOrThrow(); + return extensionUsagesTableBuilder.buildOrThrow(); + } + private ImmutableBiMap calculateUniqueNameForUsedExtensionId( + ImmutableTable extensionUsagesById) { // Calculate a unique name for each used extension id. BiMap extensionUniqueNames = HashBiMap.create(); for (ModuleExtensionId id : extensionUsagesById.rowKeySet()) { // Ensure that the resulting extension name (and thus the repository names derived from it) do // not start with a tilde. RepositoryName repository = id.getBzlFileLabel().getRepository(); - String nonEmptyRepoPart; - if (repository.isMain()) { - nonEmptyRepoPart = "_main"; - } else { - nonEmptyRepoPart = repository.getName(); - } + String nonEmptyRepoPart = repository.isMain() ? "_main" : repository.getName(); String bestName = nonEmptyRepoPart + "~" + id.getExtensionName(); if (extensionUniqueNames.putIfAbsent(bestName, id) == null) { continue; @@ -119,12 +257,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) suffix++; } } + return ImmutableBiMap.copyOf(extensionUniqueNames); + } - return BazelDepGraphValue.create( - depGraph, - canonicalRepoNameLookup, - depGraph.values().stream().map(AbridgedModule::from).collect(toImmutableList()), - extensionUsagesById, - ImmutableMap.copyOf(extensionUniqueNames.inverse())); + static class BazelDepGraphFunctionException extends SkyFunctionException { + BazelDepGraphFunctionException(ExternalDepsException e, Transience transience) { + super(e, transience); + } } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java new file mode 100644 index 00000000000000..cfeb31fa0f57bb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -0,0 +1,122 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.devtools.build.lib.bazel.bzlmod.GsonTypeAdapterUtil.LOCKFILE_GSON; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.FileValue; +import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction.BazelDepGraphFunctionException; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.cmdline.LabelConstants; +import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; +import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; +import com.google.devtools.build.lib.vfs.FileSystemUtils; +import com.google.devtools.build.lib.vfs.Path; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionException.Transience; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.google.gson.JsonIOException; +import java.io.FileNotFoundException; +import java.io.IOException; +import javax.annotation.Nullable; + +/** Reads the contents of the lock file into its value. */ +public class BazelLockFileFunction implements SkyFunction { + + public static final Precomputed LOCKFILE_MODE = new Precomputed<>("lockfile_mode"); + + private final Path rootDirectory; + + private static final BzlmodFlagsAndEnvVars EMPTY_FLAGS = + BzlmodFlagsAndEnvVars.create( + ImmutableList.of(), ImmutableMap.of(), ImmutableList.of(), "", false, "", ""); + + private static final BazelLockFileValue EMPTY_LOCKFILE = + BazelLockFileValue.create( + BazelLockFileValue.LOCK_FILE_VERSION, + "", + EMPTY_FLAGS, + ImmutableMap.of(), + ImmutableMap.of()); + + public BazelLockFileFunction(Path rootDirectory) { + this.rootDirectory = rootDirectory; + } + + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) + throws SkyFunctionException, InterruptedException { + RootedPath lockfilePath = + RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); + + // Add dependency on the lockfile to recognize changes to it + if (env.getValue(FileValue.key(lockfilePath)) == null) { + return null; + } + + BazelLockFileValue bazelLockFileValue; + try { + String json = FileSystemUtils.readContent(lockfilePath.asPath(), UTF_8); + bazelLockFileValue = LOCKFILE_GSON.fromJson(json, BazelLockFileValue.class); + } catch (FileNotFoundException e) { + bazelLockFileValue = EMPTY_LOCKFILE; + } catch (IOException ex) { + throw new JsonIOException("Failed to read or parse module-lock file", ex); + } + return bazelLockFileValue; + } + + /** + * Updates the stored module in the lock file (ModuleHash, Flags & Dependency graph) + * + * @param moduleFileHash The hash of the current module file + * @param resolvedDepGraph The resolved dependency graph from the module file + */ + public static void updateLockedModule( + Path rootDirectory, + String moduleFileHash, + BzlmodFlagsAndEnvVars flags, + ImmutableMap localOverrideHashes, + ImmutableMap resolvedDepGraph) + throws BazelDepGraphFunctionException { + RootedPath lockfilePath = + RootedPath.toRootedPath(Root.fromPath(rootDirectory), LabelConstants.MODULE_LOCKFILE_NAME); + + BazelLockFileValue value = + BazelLockFileValue.create( + BazelLockFileValue.LOCK_FILE_VERSION, + moduleFileHash, + flags, + localOverrideHashes, + resolvedDepGraph); + try { + FileSystemUtils.writeContent(lockfilePath.asPath(), UTF_8, LOCKFILE_GSON.toJson(value)); + } catch (IOException e) { + throw new BazelDepGraphFunctionException( + ExternalDepsException.withCauseAndMessage( + Code.BAD_MODULE, e, "Unable to update module-lock file"), + Transience.PERSISTENT); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java new file mode 100644 index 00000000000000..6f32405f61dd2c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -0,0 +1,90 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; +import java.util.ArrayList; +import java.util.Map; + +/** + * The result of reading the lockfile. Contains the lockfile version, module hash, definitions of + * module repositories, post-resolution dependency graph and module extensions data (ID, hash, + * definition, usages) + */ +@AutoValue +@GenerateTypeAdapter +public abstract class BazelLockFileValue implements SkyValue { + + public static final int LOCK_FILE_VERSION = 1; + + @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_LOCK_FILE; + + public static BazelLockFileValue create( + int lockFileVersion, + String moduleFileHash, + BzlmodFlagsAndEnvVars flags, + ImmutableMap localOverrideHashes, + ImmutableMap moduleDepGraph) { + return new AutoValue_BazelLockFileValue( + lockFileVersion, moduleFileHash, flags, localOverrideHashes, moduleDepGraph); + } + + /** Current version of the lock file */ + public abstract int getLockFileVersion(); + + /** Hash of the Module file */ + public abstract String getModuleFileHash(); + + /** Command line flags and environment variables that can affect the resolution */ + public abstract BzlmodFlagsAndEnvVars getFlags(); + + /** Module hash of each local path override in the root module file */ + public abstract ImmutableMap getLocalOverrideHashes(); + + /** The post-selection dep graph retrieved from the lock file. */ + public abstract ImmutableMap getModuleDepGraph(); + + /** Returns the difference between the lockfile and the current module & flags */ + public ArrayList getDiffLockfile( + String moduleFileHash, + ImmutableMap localOverrideHashes, + BzlmodFlagsAndEnvVars flags) { + ArrayList diffLockfile = new ArrayList<>(); + if (!moduleFileHash.equals(getModuleFileHash())) { + diffLockfile.add("the root MODULE.bazel has been modified"); + } + diffLockfile.addAll(getFlags().getDiffFlags(flags)); + + for (Map.Entry entry : localOverrideHashes.entrySet()) { + String currentValue = entry.getValue(); + String lockfileValue = getLocalOverrideHashes().get(entry.getKey()); + // If the lockfile value is null, the module hash would be different anyway + if (lockfileValue != null && !currentValue.equals(lockfileValue)) { + diffLockfile.add( + "The MODULE.bazel file has changed for the overriden module: " + entry.getKey()); + } + } + + return diffLockfile; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java index c44ca83ea57bc7..3c9f1c825e6c97 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java @@ -60,7 +60,7 @@ public class BazelModuleResolutionFunction implements SkyFunction { public static final Precomputed> ALLOWED_YANKED_VERSIONS = new Precomputed<>("allowed_yanked_versions"); - private static final String BZLMOD_ALLOWED_YANKED_VERSIONS_ENV = "BZLMOD_ALLOW_YANKED_VERSIONS"; + public static final String BZLMOD_ALLOWED_YANKED_VERSIONS_ENV = "BZLMOD_ALLOW_YANKED_VERSIONS"; @Override @Nullable @@ -109,7 +109,35 @@ public SkyValue compute(SkyKey skyKey, Environment env) Objects.requireNonNull(ALLOWED_YANKED_VERSIONS.get(env))), env.getListener()); - return selectionResultValue; + // Add repo spec to each module and remove registry + try { + ImmutableMap.Builder mapBuilder = ImmutableMap.builder(); + for (Map.Entry entry : resolvedDepGraph.entrySet()) { + Module module = entry.getValue(); + // Only change modules with registry (not overridden) + if (module.getRegistry() != null) { + RepoSpec moduleRepoSpec = + module + .getRegistry() + .getRepoSpec(module.getKey(), module.getCanonicalRepoName(), env.getListener()); + ModuleOverride override = root.getOverrides().get(entry.getKey().getName()); + moduleRepoSpec = maybeAppendAdditionalPatches(moduleRepoSpec, override); + module = module.toBuilder().setRepoSpec(moduleRepoSpec).setRegistry(null).build(); + } + mapBuilder.put(entry.getKey(), module); + } + resolvedDepGraph = mapBuilder.buildOrThrow(); + } catch (IOException e) { + throw new BazelModuleResolutionFunctionException( + ExternalDepsException.withMessage( + Code.ERROR_ACCESSING_REGISTRY, + "Unable to get module repo spec from registry: %s", + e.getMessage()), + Transience.PERSISTENT); + } + + return BazelModuleResolutionValue.create( + resolvedDepGraph, selectionResultValue.getUnprunedDepGraph()); } private static void verifyRootModuleDirectDepsAreAccurate( @@ -321,6 +349,26 @@ private void verifyYankedVersions( } } + private RepoSpec maybeAppendAdditionalPatches(RepoSpec repoSpec, ModuleOverride override) { + if (!(override instanceof SingleVersionOverride)) { + return repoSpec; + } + SingleVersionOverride singleVersion = (SingleVersionOverride) override; + if (singleVersion.getPatches().isEmpty()) { + return repoSpec; + } + ImmutableMap.Builder attrBuilder = ImmutableMap.builder(); + attrBuilder.putAll(repoSpec.attributes().attributes()); + attrBuilder.put("patches", singleVersion.getPatches()); + attrBuilder.put("patch_cmds", singleVersion.getPatchCmds()); + attrBuilder.put("patch_args", ImmutableList.of("-p" + singleVersion.getPatchStrip())); + return RepoSpec.builder() + .setBzlFile(repoSpec.bzlFile()) + .setRuleClassName(repoSpec.ruleClassName()) + .setAttributes(AttributeValues.create(attrBuilder.buildOrThrow())) + .build(); + } + static class BazelModuleResolutionFunctionException extends SkyFunctionException { BazelModuleResolutionFunctionException(ExternalDepsException e, Transience transience) { super(e, transience); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java new file mode 100644 index 00000000000000..f5f483f716c04b --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java @@ -0,0 +1,94 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; +import java.util.ArrayList; + +/** Stores the values of flags and environment variables that affect the resolution */ +@AutoValue +@GenerateTypeAdapter +abstract class BzlmodFlagsAndEnvVars { + + public static BzlmodFlagsAndEnvVars create( + ImmutableList registries, + ImmutableMap moduleOverrides, + ImmutableList yankedVersions, + String envVarYankedVersions, + boolean ignoreDevDeps, + String directDepsMode, + String compatabilityMode) { + return new AutoValue_BzlmodFlagsAndEnvVars( + registries, + moduleOverrides, + yankedVersions, + envVarYankedVersions, + ignoreDevDeps, + directDepsMode, + compatabilityMode); + } + + /** Registries provided via command line */ + public abstract ImmutableList cmdRegistries(); + + /** ModulesOverride provided via command line */ + public abstract ImmutableMap cmdModuleOverrides(); + + /** Allowed yanked version in the dependency graph */ + public abstract ImmutableList allowedYankedVersions(); + + /** Allowed yanked version in the dependency graph from environment variables */ + public abstract String envVarAllowedYankedVersions(); + + /** Whether to ignore things declared as dev dependencies or not */ + public abstract boolean ignoreDevDependency(); + + /** Error level of direct dependencies check */ + public abstract String directDependenciesMode(); + + /** Error level of bazel compatability check */ + public abstract String compatibilityMode(); + + public ArrayList getDiffFlags(BzlmodFlagsAndEnvVars flags) { + ArrayList diffFlags = new ArrayList<>(); + if (!flags.cmdRegistries().equals(cmdRegistries())) { + diffFlags.add("the value of --registry flag has been modified"); + } + if (!flags.cmdModuleOverrides().equals(cmdModuleOverrides())) { + diffFlags.add("the value of --override_module flag has been modified"); + } + if (!flags.allowedYankedVersions().equals(allowedYankedVersions())) { + diffFlags.add("the value of --allow_yanked_versions flag has been modified"); + } + if (!flags.envVarAllowedYankedVersions().equals(envVarAllowedYankedVersions())) { + diffFlags.add( + "the value of BZLMOD_ALLOW_YANKED_VERSIONS environment variable has been modified"); + } + if (flags.ignoreDevDependency() != ignoreDevDependency()) { + diffFlags.add("the value of --ignore_dev_dependency flag has been modified"); + } + if (!flags.directDependenciesMode().equals(directDependenciesMode())) { + diffFlags.add("the value of --check_direct_dependencies flag has been modified"); + } + if (!flags.compatibilityMode().equals(compatibilityMode())) { + diffFlags.add("the value of --check_bazel_compatibility flag has been modified"); + } + return diffFlags; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/DelegateTypeAdapterFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/DelegateTypeAdapterFactory.java new file mode 100644 index 00000000000000..d791789dc46bbb --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/DelegateTypeAdapterFactory.java @@ -0,0 +1,126 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.common.collect.ImmutableBiMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import java.io.IOException; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import javax.annotation.Nullable; +import net.starlark.java.eval.Dict; + +/** + * Creates Gson type adapters for parameterized types by using a delegate parameterized type that + * already has a registered type adapter factory. + */ +public final class DelegateTypeAdapterFactory + implements TypeAdapterFactory { + private final Class rawType; + private final Class intermediateToDelegateType; + private final Class delegateType; + private final Function rawToDelegate; + private final Function delegateToRaw; + + private DelegateTypeAdapterFactory( + Class rawType, + Class intermediateToDelegateType, + Class delegateType, + Function rawToDelegate, + Function delegateToRaw) { + this.rawType = rawType; + this.intermediateToDelegateType = intermediateToDelegateType; + this.delegateType = delegateType; + this.rawToDelegate = rawToDelegate; + this.delegateToRaw = delegateToRaw; + } + + public static final TypeAdapterFactory IMMUTABLE_MAP = + new DelegateTypeAdapterFactory<>( + ImmutableMap.class, + Map.class, + LinkedHashMap.class, + raw -> new LinkedHashMap<>((Map) raw), + delegate -> ImmutableMap.copyOf((Map) delegate)); + + public static final TypeAdapterFactory IMMUTABLE_BIMAP = + new DelegateTypeAdapterFactory<>( + ImmutableBiMap.class, + Map.class, + LinkedHashMap.class, + raw -> new LinkedHashMap<>((Map) raw), + delegate -> ImmutableBiMap.copyOf((Map) delegate)); + + public static final TypeAdapterFactory DICT = + new DelegateTypeAdapterFactory<>( + Dict.class, + Map.class, + LinkedHashMap.class, + raw -> new LinkedHashMap<>((Map) raw), + delegate -> Dict.immutableCopyOf((Map) delegate)); + + public static final TypeAdapterFactory IMMUTABLE_LIST = + new DelegateTypeAdapterFactory<>( + ImmutableList.class, + List.class, + ArrayList.class, + raw -> new ArrayList<>((List) raw), + delegate -> ImmutableList.copyOf((List) delegate)); + + @SuppressWarnings("unchecked") + @Override + @Nullable + public TypeAdapter create(Gson gson, TypeToken typeToken) { + Type type = typeToken.getType(); + if (typeToken.getRawType() != rawType || !(type instanceof ParameterizedType)) { + return null; + } + + com.google.common.reflect.TypeToken betterToken = + com.google.common.reflect.TypeToken.of(typeToken.getType()); + final TypeAdapter delegateAdapter = + (TypeAdapter) + gson.getAdapter( + TypeToken.get( + betterToken + .getSupertype((Class) intermediateToDelegateType) + .getSubtype(delegateType) + .getType())); + return new TypeAdapter() { + @Override + public void write(JsonWriter out, T value) throws IOException { + delegateAdapter.write(out, rawToDelegate.apply((R) value)); + } + + @Override + public T read(JsonReader in) throws IOException { + return (T) delegateToRaw.apply((D) delegateAdapter.read(in)); + } + }; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GitOverride.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GitOverride.java index 6270a43fea7658..4b264cf098be32 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GitOverride.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GitOverride.java @@ -62,7 +62,7 @@ public RepoSpec getRepoSpec(RepositoryName repoName) { return RepoSpec.builder() .setBzlFile("@bazel_tools//tools/build_defs/repo:git.bzl") .setRuleClassName("git_repository") - .setAttributes(attrBuilder.buildOrThrow()) + .setAttributes(AttributeValues.create(attrBuilder.buildOrThrow())) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java new file mode 100644 index 00000000000000..3cb6d22954126a --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java @@ -0,0 +1,104 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.DICT; +import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_BIMAP; +import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_LIST; +import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_MAP; + +import com.google.common.base.Splitter; +import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; +import java.io.IOException; +import java.util.List; + +/** + * Utility class to hold type adapters and helper methods to get gson registered with type adapters + */ +public final class GsonTypeAdapterUtil { + + public static final TypeAdapter VERSION_TYPE_ADAPTER = + new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, Version version) throws IOException { + jsonWriter.value(version.toString()); + } + + @Override + public Version read(JsonReader jsonReader) throws IOException { + Version version; + String versionString = jsonReader.nextString(); + try { + version = Version.parse(versionString); + } catch (ParseException e) { + throw new JsonParseException( + String.format("Unable to parse Version %s from the lockfile", versionString), e); + } + return version; + } + }; + + public static final TypeAdapter MODULE_KEY_TYPE_ADAPTER = + new TypeAdapter<>() { + @Override + public void write(JsonWriter jsonWriter, ModuleKey moduleKey) throws IOException { + jsonWriter.value(moduleKey.toString()); + } + + @Override + public ModuleKey read(JsonReader jsonReader) throws IOException { + String jsonString = jsonReader.nextString(); + if (jsonString.equals("")) { + return ModuleKey.ROOT; + } + List parts = Splitter.on('@').splitToList(jsonString); + if (parts.get(1).equals("_")) { + return ModuleKey.create(parts.get(0), Version.EMPTY); + } + + Version version; + try { + version = Version.parse(parts.get(1)); + } catch (ParseException e) { + throw new JsonParseException( + String.format("Unable to parse ModuleKey %s version from the lockfile", jsonString), + e); + } + return ModuleKey.create(parts.get(0), version); + } + }; + + public static final Gson LOCKFILE_GSON = + new GsonBuilder() + .setPrettyPrinting() + .registerTypeAdapterFactory(GenerateTypeAdapter.FACTORY) + .registerTypeAdapterFactory(DICT) + .registerTypeAdapterFactory(IMMUTABLE_MAP) + .registerTypeAdapterFactory(IMMUTABLE_LIST) + .registerTypeAdapterFactory(IMMUTABLE_BIMAP) + .registerTypeAdapter(Version.class, VERSION_TYPE_ADAPTER) + .registerTypeAdapter(ModuleKey.class, MODULE_KEY_TYPE_ADAPTER) + .registerTypeAdapter(AttributeValues.class, new AttributeValuesAdapter()) + .create(); + + private GsonTypeAdapterUtil() {} +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index 83266c3032518f..46be96ffef45f4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -194,8 +194,9 @@ private RepoSpec createLocalPathRepoSpec( return RepoSpec.builder() .setRuleClassName("local_repository") .setAttributes( - ImmutableMap.of( - "name", repoName.getName(), "path", PathFragment.create(path).toString())) + AttributeValues.create( + ImmutableMap.of( + "name", repoName.getName(), "path", PathFragment.create(path).toString()))) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LocalPathOverride.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LocalPathOverride.java index 28e4cc4e476b55..33f5f64caf33d0 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LocalPathOverride.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LocalPathOverride.java @@ -35,7 +35,8 @@ public static LocalPathOverride create(String path) { public RepoSpec getRepoSpec(RepositoryName repoName) { return RepoSpec.builder() .setRuleClassName("local_repository") - .setAttributes(ImmutableMap.of("name", repoName.getName(), "path", getPath())) + .setAttributes( + AttributeValues.create(ImmutableMap.of("name", repoName.getName(), "path", getPath()))) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java index 2952548e851bd5..ca56fb4ac28bd5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Module.java @@ -22,6 +22,7 @@ import com.google.devtools.build.lib.cmdline.RepositoryMapping; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import java.util.Map; import java.util.Optional; import java.util.function.UnaryOperator; @@ -35,6 +36,7 @@ * discovery but before selection, or when there's a multiple_version_override in play). */ @AutoValue +@GenerateTypeAdapter public abstract class Module { /** @@ -136,13 +138,22 @@ public final RepositoryMapping getRepoMappingWithBazelDepsOnly() { return RepositoryMapping.create(mapping.buildOrThrow(), getCanonicalRepoName()); } + // TODO(salmasamy) create two modules (One with registry, one with repospec and only necessary + // things for lockfile) /** * The registry where this module came from. Must be null iff the module has a {@link - * NonRegistryOverride}. + * NonRegistryOverride}. Set to null after running selection and verifying yanked versions. */ @Nullable public abstract Registry getRegistry(); + /** + * The repo spec for this module (information about the attributes of its repository rule) Filled + * after running selection to avoid extra calls to the registry. + */ + @Nullable + public abstract RepoSpec getRepoSpec(); + /** The module extensions used in this module. */ public abstract ImmutableList getExtensionUsages(); @@ -186,6 +197,8 @@ public abstract static class Builder { /** Optional; defaults to {@link #setName}. */ public abstract Builder setRepoName(String value); + public abstract Builder setBazelCompatibility(ImmutableList value); + abstract ImmutableList.Builder bazelCompatibilityBuilder(); @CanIgnoreReturnValue @@ -194,6 +207,8 @@ public final Builder addBazelCompatibilityValues(Iterable values) { return this; } + public abstract Builder setExecutionPlatformsToRegister(ImmutableList value); + abstract ImmutableList.Builder executionPlatformsToRegisterBuilder(); @CanIgnoreReturnValue @@ -202,6 +217,8 @@ public final Builder addExecutionPlatformsToRegister(Iterable values) { return this; } + public abstract Builder setToolchainsToRegister(ImmutableList value); + abstract ImmutableList.Builder toolchainsToRegisterBuilder(); @CanIgnoreReturnValue @@ -232,6 +249,8 @@ public Builder addOriginalDep(String depRepoName, ModuleKey depKey) { public abstract Builder setRegistry(Registry value); + public abstract Builder setRepoSpec(RepoSpec value); + public abstract Builder setExtensionUsages(ImmutableList value); abstract ImmutableList.Builder extensionUsagesBuilder(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java index 9266ef39a5ec94..64be185428b300 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import net.starlark.java.syntax.Location; /** @@ -25,6 +26,7 @@ * information pertinent to the proxy object returned from the {@code use_extension} call. */ @AutoValue +@GenerateTypeAdapter public abstract class ModuleExtensionUsage { /** An unresolved label pointing to the Starlark file where the module extension is defined. */ public abstract String getExtensionBzlFile(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java index a7c1a0cbb51d7c..d9714602c8f71c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java @@ -30,6 +30,7 @@ import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; +import com.google.devtools.build.lib.util.Fingerprint; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Root; @@ -108,6 +109,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (getModuleFileResult == null) { return null; } + String moduleFileHash = + new Fingerprint().addBytes(getModuleFileResult.moduleFileContents).hexDigestAndReset(); ModuleFileGlobals moduleFileGlobals = execModuleFile( @@ -139,7 +142,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) throw errorf(Code.BAD_MODULE, "The MODULE.bazel file of %s declares overrides", moduleKey); } - return NonRootModuleFileValue.create(module); + return NonRootModuleFileValue.create(module, moduleFileHash); } @Nullable @@ -152,6 +155,7 @@ private SkyValue computeForRootModule(StarlarkSemantics starlarkSemantics, Envir return null; } byte[] moduleFile = readFile(moduleFilePath.asPath()); + String moduleFileHash = new Fingerprint().addBytes(moduleFile).hexDigestAndReset(); ModuleFileGlobals moduleFileGlobals = execModuleFile( moduleFile, @@ -184,7 +188,7 @@ private SkyValue computeForRootModule(StarlarkSemantics starlarkSemantics, Envir name -> ModuleKey.create(name, Version.EMPTY).getCanonicalRepoName(), name -> name)); return RootModuleFileValue.create( - module, overrides, nonRegistryOverrideCanonicalRepoNameLookup); + module, moduleFileHash, overrides, nonRegistryOverrideCanonicalRepoNameLookup); } private ModuleFileGlobals execModuleFile( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java index d48359d212a605..7e1909a82c5272 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java @@ -467,7 +467,7 @@ public void call(Dict kwargs, StarlarkThread thread) { tags.add( Tag.builder() .setTagName(tagName) - .setAttributeValues(kwargs) + .setAttributeValues(AttributeValues.create(kwargs)) .setDevDependency(devDependency) .setLocation(thread.getCallerLocation()) .build()); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java index c6202fefdeb088..48d47e8e87ac02 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileValue.java @@ -40,12 +40,15 @@ public abstract class ModuleFileValue implements SkyValue { */ public abstract Module getModule(); + /** The hash string of Module.bazel (using SHA256) */ + public abstract String getModuleFileHash(); + /** The {@link ModuleFileValue} for non-root modules. */ @AutoValue public abstract static class NonRootModuleFileValue extends ModuleFileValue { - public static NonRootModuleFileValue create(Module module) { - return new AutoValue_ModuleFileValue_NonRootModuleFileValue(module); + public static NonRootModuleFileValue create(Module module, String moduleFileHash) { + return new AutoValue_ModuleFileValue_NonRootModuleFileValue(module, moduleFileHash); } } @@ -70,10 +73,11 @@ public abstract static class RootModuleFileValue extends ModuleFileValue { public static RootModuleFileValue create( Module module, + String moduleFileHash, ImmutableMap overrides, ImmutableMap nonRegistryOverrideCanonicalRepoNameLookup) { return new AutoValue_ModuleFileValue_RootModuleFileValue( - module, overrides, nonRegistryOverrideCanonicalRepoNameLookup); + module, moduleFileHash, overrides, nonRegistryOverrideCanonicalRepoNameLookup); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java index 71324a875cb169..f5f2e6f851c8b7 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java @@ -15,24 +15,26 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableMap; -import java.util.Optional; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; +import javax.annotation.Nullable; /** * A class holding information about the attributes of a repository rule and where the rule class is * defined. */ @AutoValue +@GenerateTypeAdapter public abstract class RepoSpec { /** * The label string for the bzl file this repository rule is defined in, empty for native rule. */ - public abstract Optional bzlFile(); + @Nullable + public abstract String bzlFile(); public abstract String ruleClassName(); - public abstract ImmutableMap attributes(); + public abstract AttributeValues attributes(); public static Builder builder() { return new AutoValue_RepoSpec.Builder(); @@ -43,17 +45,15 @@ public static Builder builder() { public abstract static class Builder { public abstract Builder setBzlFile(String bzlFile); - public abstract Builder setBzlFile(Optional bzlFile); - public abstract Builder setRuleClassName(String name); - public abstract Builder setAttributes(ImmutableMap attributes); + public abstract Builder setAttributes(AttributeValues attributes); public abstract RepoSpec build(); } public boolean isNativeRepoRule() { - return !bzlFile().isPresent(); + return bzlFile() == null; } /** @@ -61,6 +61,6 @@ public boolean isNativeRepoRule() { * repo rule: //:repo.bzl%my_repo */ public String getRuleClass() { - return bzlFile().map(f -> f + "%").orElse("") + ruleClassName(); + return (bzlFile() == null ? "" : bzlFile() + "%") + ruleClassName(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Tag.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Tag.java index 498aacd427c49a..4ee4d6db356c92 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Tag.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Tag.java @@ -16,7 +16,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.auto.value.AutoValue; -import net.starlark.java.eval.Dict; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import net.starlark.java.syntax.Location; /** @@ -26,12 +26,13 @@ * not when the tag is created, which is during module discovery). */ @AutoValue +@GenerateTypeAdapter public abstract class Tag { public abstract String getTagName(); /** All keyword arguments supplied to the tag instance. */ - public abstract Dict getAttributeValues(); + public abstract AttributeValues getAttributeValues(); /** Whether this tag was created using a proxy created with dev_dependency = True. */ public abstract boolean isDevDependency(); @@ -49,7 +50,7 @@ public abstract static class Builder { public abstract Builder setTagName(String value); - public abstract Builder setAttributeValues(Dict value); + public abstract Builder setAttributeValues(AttributeValues value); public abstract Builder setDevDependency(boolean value); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/TypeCheckedTag.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/TypeCheckedTag.java index 54261a2349f863..443321b711604e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/TypeCheckedTag.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/TypeCheckedTag.java @@ -46,7 +46,7 @@ private TypeCheckedTag(TagClass tagClass, Object[] attrValues, boolean devDepend public static TypeCheckedTag create(TagClass tagClass, Tag tag, LabelConverter labelConverter) throws ExternalDepsException { Object[] attrValues = new Object[tagClass.getAttributes().size()]; - for (Map.Entry attrValue : tag.getAttributeValues().entrySet()) { + for (Map.Entry attrValue : tag.getAttributeValues().attributes().entrySet()) { Integer attrIndex = tagClass.getAttributeIndices().get(attrValue.getKey()); if (attrIndex == null) { throw ExternalDepsException.withMessage( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java index c860cd7e87551f..f2cbea78e44119 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java @@ -275,6 +275,19 @@ public class RepositoryOptions extends OptionsBase { + " warning when mismatch detected.") public BazelCompatibilityMode bazelCompatibilityMode; + @Option( + name = "lockfile_mode", + defaultValue = "off", // TODO(salmasamy) later will be changed to 'update' + converter = LockfileMode.Converter.class, + documentationCategory = OptionDocumentationCategory.BZLMOD, + effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS}, + help = + "Specifies how and whether or not to use the lockfile. Valid values are `update` to" + + " use the lockfile and update it if there are changes, `error` to use the lockfile" + + " but throw an error if it's not up-to-date, or `off` to neither read from or write" + + " to the lockfile.") + public LockfileMode lockfileMode; + /** An enum for specifying different modes for checking direct dependency accuracy. */ public enum CheckDirectDepsMode { OFF, // Don't check direct dependency accuracy. @@ -288,6 +301,7 @@ public Converter() { } } } + /** An enum for specifying different modes for bazel compatibility check. */ public enum BazelCompatibilityMode { ERROR, // Check and throw an error when mismatched. @@ -302,6 +316,20 @@ public Converter() { } } + /** An enum for specifying how to use the lockfile. */ + public enum LockfileMode { + OFF, // Don't use the lockfile at all. + UPDATE, // Update the lockfile when it mismatches the module. + ERROR; // Throw an error when it mismatches the module. + + /** Converts to {@link BazelLockfileMode}. */ + public static class Converter extends EnumConverter { + public Converter() { + super(LockfileMode.class, "Lockfile mode"); + } + } + } + /** * Converts from an equals-separated pair of strings into RepositoryName->PathFragment mapping. */ diff --git a/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java b/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java index b133f40ca94927..783396f7fe41bd 100644 --- a/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java +++ b/src/main/java/com/google/devtools/build/lib/cmdline/LabelConstants.java @@ -44,6 +44,9 @@ public class LabelConstants { public static final PathFragment WORKSPACE_DOT_BAZEL_FILE_NAME = PathFragment.create("WORKSPACE.bazel"); public static final PathFragment MODULE_DOT_BAZEL_FILE_NAME = PathFragment.create("MODULE.bazel"); + + public static final PathFragment MODULE_LOCKFILE_NAME = PathFragment.create("MODULE.bazel.lock"); + public static final String DEFAULT_REPOSITORY_DIRECTORY = "__main__"; // With this prefix, non-main repositories are symlinked under diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java index fe4977fe1e001b..e4942e3efadb7d 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java @@ -27,17 +27,13 @@ import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; -import com.google.devtools.build.lib.bazel.bzlmod.ModuleOverride; import com.google.devtools.build.lib.bazel.bzlmod.NonRegistryOverride; -import com.google.devtools.build.lib.bazel.bzlmod.Registry; import com.google.devtools.build.lib.bazel.bzlmod.RepoSpec; import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalValue; -import com.google.devtools.build.lib.bazel.bzlmod.SingleVersionOverride; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.RepositoryMapping; import com.google.devtools.build.lib.cmdline.RepositoryName; -import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.packages.NoSuchPackageException; import com.google.devtools.build.lib.packages.Package; import com.google.devtools.build.lib.packages.Rule; @@ -52,7 +48,6 @@ import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import java.io.IOException; import java.util.Map.Entry; import java.util.Optional; import javax.annotation.Nullable; @@ -104,38 +99,28 @@ public SkyValue compute(SkyKey skyKey, Environment env) } RepositoryName repositoryName = ((BzlmodRepoRuleValue.Key) skyKey).argument(); - BazelDepGraphValue bazelDepGraphValue; - // Try to find a module from this repository name. If not found, try to find - // an extension. If none found, it is an invalid name (return not found) - - // Look for the repo from Bazel module generated repos. - try { - // Step 1: Look for repositories defined by non-registry overrides. - Optional repoSpec = checkRepoFromNonRegistryOverrides(root, repositoryName); - if (repoSpec.isPresent()) { - return createRuleFromSpec(repoSpec.get(), starlarkSemantics, env); - } + // Step 1: Look for repositories defined by non-registry overrides. + Optional repoSpec = checkRepoFromNonRegistryOverrides(root, repositoryName); + if (repoSpec.isPresent()) { + return createRuleFromSpec(repoSpec.get(), starlarkSemantics, env); + } - // BazelDepGraphValue is affected by repos found in Step 1, therefore it should NOT - // be requested in Step 1 to avoid cycle dependency. - bazelDepGraphValue = (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY); - if (env.valuesMissing()) { - return null; - } + // BazelDepGraphValue is affected by repos found in Step 1, therefore it should NOT + // be requested in Step 1 to avoid cycle dependency. + BazelDepGraphValue bazelDepGraphValue = + (BazelDepGraphValue) env.getValue(BazelDepGraphValue.KEY); + if (env.valuesMissing()) { + return null; + } - // Step 2: Look for repositories derived from Bazel Modules. - repoSpec = - checkRepoFromBazelModules( - bazelDepGraphValue, root.getOverrides(), env.getListener(), repositoryName); - if (repoSpec.isPresent()) { - return createRuleFromSpec(repoSpec.get(), starlarkSemantics, env); - } - } catch (IOException e) { - throw new BzlmodRepoRuleFunctionException(e, Transience.PERSISTENT); + // Step 2: Look for repositories derived from Bazel Modules. + repoSpec = checkRepoFromBazelModules(bazelDepGraphValue, repositoryName); + if (repoSpec.isPresent()) { + return createRuleFromSpec(repoSpec.get(), starlarkSemantics, env); } - // Otherwise, look for the repo from module extension evaluation results. + // Step 3: look for the repo from module extension evaluation results. Optional extensionId = bazelDepGraphValue.getExtensionUniqueNames().entrySet().stream() .filter(e -> repositoryName.getName().startsWith(e.getValue() + "~")) @@ -173,41 +158,14 @@ private static Optional checkRepoFromNonRegistryOverrides( } private Optional checkRepoFromBazelModules( - BazelDepGraphValue bazelDepGraphValue, - ImmutableMap overrides, - ExtendedEventHandler eventListener, - RepositoryName repositoryName) - throws InterruptedException, IOException { + BazelDepGraphValue bazelDepGraphValue, RepositoryName repositoryName) { ModuleKey moduleKey = bazelDepGraphValue.getCanonicalRepoNameLookup().get(repositoryName); if (moduleKey == null) { return Optional.empty(); } com.google.devtools.build.lib.bazel.bzlmod.Module module = bazelDepGraphValue.getDepGraph().get(moduleKey); - Registry registry = checkNotNull(module.getRegistry()); - RepoSpec repoSpec = registry.getRepoSpec(moduleKey, repositoryName, eventListener); - repoSpec = maybeAppendAdditionalPatches(repoSpec, overrides.get(moduleKey.getName())); - return Optional.of(repoSpec); - } - - private RepoSpec maybeAppendAdditionalPatches(RepoSpec repoSpec, ModuleOverride override) { - if (!(override instanceof SingleVersionOverride)) { - return repoSpec; - } - SingleVersionOverride singleVersion = (SingleVersionOverride) override; - if (singleVersion.getPatches().isEmpty()) { - return repoSpec; - } - ImmutableMap.Builder attrBuilder = ImmutableMap.builder(); - attrBuilder.putAll(repoSpec.attributes()); - attrBuilder.put("patches", singleVersion.getPatches()); - attrBuilder.put("patch_cmds", singleVersion.getPatchCmds()); - attrBuilder.put("patch_args", ImmutableList.of("-p" + singleVersion.getPatchStrip())); - return RepoSpec.builder() - .setBzlFile(repoSpec.bzlFile()) - .setRuleClassName(repoSpec.ruleClassName()) - .setAttributes(attrBuilder.buildOrThrow()) - .build(); + return Optional.of(checkNotNull(module.getRepoSpec())); } @Nullable @@ -225,7 +183,7 @@ private BzlmodRepoRuleValue createRuleFromSpec( ruleClass = ruleClassProvider.getRuleClassMap().get(repoSpec.ruleClassName()); } else { ImmutableMap loadedModules = - loadBzlModules(env, repoSpec.bzlFile().get(), starlarkSemantics); + loadBzlModules(env, repoSpec.bzlFile(), starlarkSemantics); if (env.valuesMissing()) { return null; } @@ -242,7 +200,7 @@ private BzlmodRepoRuleValue createRuleFromSpec( env.getListener(), "BzlmodRepoRuleFunction.createRule", ruleClass, - repoSpec.attributes()); + repoSpec.attributes().attributes()); return new BzlmodRepoRuleValue(rule.getPackage(), rule.getName()); } catch (InvalidRuleException e) { throw new BzlmodRepoRuleFunctionException(e, Transience.PERSISTENT); @@ -301,7 +259,7 @@ private ImmutableMap loadBzlModules( private RuleClass getStarlarkRuleClass( RepoSpec repoSpec, ImmutableMap loadedModules) throws BzlmodRepoRuleFunctionException { - Object object = loadedModules.get(repoSpec.bzlFile().get()).getGlobal(repoSpec.ruleClassName()); + Object object = loadedModules.get(repoSpec.bzlFile()).getGlobal(repoSpec.ruleClassName()); if (object instanceof RuleFunction) { return ((RuleFunction) object).getRuleClass(); } else { @@ -321,10 +279,6 @@ private static final class BzlmodRepoRuleFunctionException extends SkyFunctionEx super(e, transience); } - BzlmodRepoRuleFunctionException(IOException e, Transience transience) { - super(e, transience); - } - BzlmodRepoRuleFunctionException(EvalException e, Transience transience) { super(e, transience); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java index 863a9d472976c3..9eb8b4f11577e3 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java @@ -153,6 +153,8 @@ public final class SkyFunctions { SkyFunctionName.createNonHermetic("SINGLE_EXTENSION_EVAL"); public static final SkyFunctionName BAZEL_DEP_GRAPH = SkyFunctionName.createHermetic("BAZEL_DEP_GRAPH"); + public static final SkyFunctionName BAZEL_LOCK_FILE = + SkyFunctionName.createHermetic("BAZEL_LOCK_FILE"); public static Predicate isSkyFunction(SkyFunctionName functionName) { return key -> key.functionName().equals(functionName); diff --git a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java index 4c27d2bd2a4b44..edb5955f579b64 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/RunfilesRepoMappingManifestTest.java @@ -21,11 +21,13 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.actions.Action; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Injected; import com.google.devtools.build.lib.vfs.Path; @@ -54,6 +56,7 @@ protected ImmutableList extraPrecomputedValues() throws Exception { BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF), PrecomputedValue.injected( BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of())); } diff --git a/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java b/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java index 5acdfe65aaabcb..ea8cf9f5106196 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/StarlarkRuleTransitionProviderTest.java @@ -25,11 +25,13 @@ import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; import com.google.devtools.build.lib.analysis.util.DummyTestFragment; import com.google.devtools.build.lib.analysis.util.DummyTestFragment.DummyTestOptions; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.packages.Rule; import com.google.devtools.build.lib.packages.RuleTransitionData; @@ -74,7 +76,8 @@ protected ImmutableList extraPrecomputedValues() { PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); } @Override diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java index 2501f17dc55937..a511ce2d7d0c8e 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java @@ -18,6 +18,7 @@ import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; @@ -142,7 +143,9 @@ public ImmutableMap getSkyFunctions(BlazeDirectori directories.getWorkspace(), getBuiltinModules(directories)), SkyFunctions.BAZEL_DEP_GRAPH, - new BazelDepGraphFunction(), + new BazelDepGraphFunction(directories.getWorkspace()), + SkyFunctions.BAZEL_LOCK_FILE, + new BazelLockFileFunction(directories.getWorkspace()), SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction(), SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java index f28ef3a1596e10..373668e06a2c62 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisTestCase.java @@ -49,11 +49,13 @@ import com.google.devtools.build.lib.analysis.config.transitions.NoTransition; import com.google.devtools.build.lib.analysis.configuredtargets.InputFileConfiguredTarget; import com.google.devtools.build.lib.analysis.starlark.StarlarkTransition; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.buildtool.BuildRequestOptions; import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.cmdline.Label; @@ -241,7 +243,9 @@ protected void useRuleClassProvider(ConfiguredRuleClassProvider ruleClassProvide BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, - BazelCompatibilityMode.ERROR))) + BazelCompatibilityMode.ERROR), + PrecomputedValue.injected( + BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))) .build(ruleClassProvider, fileSystem); useConfiguration(); skyframeExecutor = createSkyframeExecutor(pkgFactory); @@ -289,7 +293,8 @@ private void reinitializeSkyframeExecutor() { BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, - BazelCompatibilityMode.WARNING))); + BazelCompatibilityMode.WARNING), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))); } /** Resets the SkyframeExecutor, as if a clean had been executed. */ diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValuesAdapterTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValuesAdapterTest.java new file mode 100644 index 00000000000000..8ae80136ea764f --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValuesAdapterTest.java @@ -0,0 +1,71 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.testutil.FoundationTestCase; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonWriter; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.Map; +import net.starlark.java.eval.Dict; +import net.starlark.java.eval.Mutability; +import net.starlark.java.eval.StarlarkInt; +import net.starlark.java.eval.StarlarkList; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class AttributeValuesAdapterTest extends FoundationTestCase { + + @Test + public void testAttributeValuesAdapter() throws IOException { + Dict.Builder dict = new Dict.Builder<>(); + Label l1 = Label.parseCanonicalUnchecked("@//foo:bar"); + Label l2 = Label.parseCanonicalUnchecked("@//foo:tar"); + dict.put("Integer", StarlarkInt.of(56)); + dict.put("Boolean", false); + dict.put("String", "Hello"); + dict.put("Label", l1); + dict.put( + "ListOfInts", StarlarkList.of(Mutability.IMMUTABLE, StarlarkInt.of(1), StarlarkInt.of(2))); + dict.put("ListOfLabels", StarlarkList.of(Mutability.IMMUTABLE, l1, l2)); + dict.put("ListOfStrings", StarlarkList.of(Mutability.IMMUTABLE, "Hello", "There!")); + Dict.Builder dictLabelString = new Dict.Builder<>(); + dictLabelString.put(l1, "Label#1"); + dictLabelString.put(l2, "Label#2"); + dict.put("DictOfLabel-String", dictLabelString.buildImmutable()); + + Dict builtDict = dict.buildImmutable(); + AttributeValuesAdapter attrAdapter = new AttributeValuesAdapter(); + String jsonString; + try (StringWriter stringWriter = new StringWriter()) { + attrAdapter.write(new JsonWriter(stringWriter), AttributeValues.create(builtDict)); + jsonString = stringWriter.toString(); + } + AttributeValues attributeValues; + try (StringReader stringReader = new StringReader(jsonString)) { + attributeValues = attrAdapter.read(new JsonReader(stringReader)); + } + + assertThat((Map) attributeValues.attributes()).containsExactlyEntriesIn(builtDict); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 34491a53c96890..3762ba587a24e6 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -78,6 +78,7 @@ java_library( "//src/test/java/com/google/devtools/build/lib/testutil", "//third_party:auto_value", "//third_party:caffeine", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", "//third_party:junit4", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index 08cb330253d37b..bcacdab1c8bb5b 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -122,28 +123,37 @@ public void setup() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, resolutionFunctionMock) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, - new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) + new ClientEnvironmentFunction( + new AtomicReference<>(ImmutableMap.of("BZLMOD_ALLOW_YANKED_VERSIONS", "")))) .buildOrThrow(), differencer); PrecomputedValue.STARLARK_SEMANTICS.set( differencer, StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( differencer, CheckDirectDepsMode.OFF); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } @Test public void createValue_basic() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + // Root depends on dep@1.0 and dep@2.0 at the same time with a multiple-version override. // Root also depends on rules_cc as a normal dep. // dep@1.0 depends on rules_java, which is overridden by a non-registry override (see below). @@ -190,7 +200,6 @@ public void createValue_basic() throws Exception { .build()) .buildOrThrow(); - // TODO we need to mock bazelModuleResolution function to return depGraph resolutionFunctionMock.setDepGraph(depGraph); EvaluationResult result = evaluator.evaluate(ImmutableList.of(BazelDepGraphValue.KEY), evaluationContext); @@ -231,6 +240,10 @@ private static ModuleExtensionUsage createModuleExtensionUsage( @Test public void createValue_moduleExtensions() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + Module root = Module.builder() .setName("root") @@ -331,6 +344,10 @@ public void createValue_moduleExtensions() throws Exception { @Test public void useExtensionBadLabelFails() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='module', version='1.0')"); + Module root = Module.builder() .addExtensionUsage(createModuleExtensionUsage("@foo//:defs.bzl", "bar")) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java new file mode 100644 index 00000000000000..9a20ae0f24a551 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -0,0 +1,408 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Suppliers; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.actions.FileValue; +import com.google.devtools.build.lib.analysis.BlazeDirectories; +import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; +import com.google.devtools.build.lib.analysis.ServerDirectories; +import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; +import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; +import com.google.devtools.build.lib.clock.BlazeClock; +import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; +import com.google.devtools.build.lib.pkgcache.PathPackageLocator; +import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction; +import com.google.devtools.build.lib.rules.repository.LocalRepositoryRule; +import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; +import com.google.devtools.build.lib.rules.repository.RepositoryFunction; +import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; +import com.google.devtools.build.lib.skyframe.BzlmodRepoRuleFunction; +import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; +import com.google.devtools.build.lib.skyframe.ExternalFilesHelper; +import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; +import com.google.devtools.build.lib.skyframe.FileFunction; +import com.google.devtools.build.lib.skyframe.FileStateFunction; +import com.google.devtools.build.lib.skyframe.PrecomputedFunction; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.starlarkbuildapi.repository.RepositoryBootstrap; +import com.google.devtools.build.lib.testutil.FoundationTestCase; +import com.google.devtools.build.lib.testutil.TestRuleClassProvider; +import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; +import com.google.devtools.build.lib.vfs.FileStateKey; +import com.google.devtools.build.lib.vfs.Root; +import com.google.devtools.build.lib.vfs.SyscallCache; +import com.google.devtools.build.skyframe.EvaluationContext; +import com.google.devtools.build.skyframe.EvaluationResult; +import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator; +import com.google.devtools.build.skyframe.MemoizingEvaluator; +import com.google.devtools.build.skyframe.RecordingDifferencer; +import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.Nullable; +import net.starlark.java.eval.StarlarkSemantics; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link BazelLockFileFunction}. */ +@RunWith(JUnit4.class) +public class BazelLockFileFunctionTest extends FoundationTestCase { + + private MemoizingEvaluator evaluator; + private RecordingDifferencer differencer; + private EvaluationContext evaluationContext; + private FakeRegistry.Factory registryFactory; + private static SkyFunctionName updateLockfileFunction; + + @Before + public void setup() throws Exception { + differencer = new SequencedRecordingDifferencer(); + registryFactory = new FakeRegistry.Factory(); + evaluationContext = + EvaluationContext.newBuilder().setNumThreads(8).setEventHandler(reporter).build(); + + AtomicReference packageLocator = + new AtomicReference<>( + new PathPackageLocator( + outputBase, + ImmutableList.of(Root.fromPath(rootDirectory)), + BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY)); + BlazeDirectories directories = + new BlazeDirectories( + new ServerDirectories(rootDirectory, outputBase, rootDirectory), + rootDirectory, + /* defaultSystemJavabase= */ null, + AnalysisMock.get().getProductName()); + ExternalFilesHelper externalFilesHelper = + ExternalFilesHelper.createForTesting( + packageLocator, + ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, + directories); + RepositoryFunction localRepositoryFunction = new LocalRepositoryFunction(); + ImmutableMap repositoryHandlers = + ImmutableMap.of(LocalRepositoryRule.NAME, localRepositoryFunction); + + ConfiguredRuleClassProvider.Builder builder = new ConfiguredRuleClassProvider.Builder(); + TestRuleClassProvider.addStandardRules(builder); + builder + .clearWorkspaceFileSuffixForTesting() + .addStarlarkBootstrap(new RepositoryBootstrap(new StarlarkRepositoryModule())); + ConfiguredRuleClassProvider ruleClassProvider = builder.build(); + + updateLockfileFunction = SkyFunctionName.createHermetic("LockfileWrite"); + + evaluator = + new InMemoryMemoizingEvaluator( + ImmutableMap.builder() + .put(FileValue.FILE, new FileFunction(packageLocator, directories)) + .put( + FileStateKey.FILE_STATE, + new FileStateFunction( + Suppliers.ofInstance( + new TimestampGranularityMonitor(BlazeClock.instance())), + SyscallCache.NO_CACHE, + externalFilesHelper)) + .put( + SkyFunctions.REPOSITORY_DIRECTORY, + new RepositoryDelegatorFunction( + repositoryHandlers, + null, + new AtomicBoolean(true), + ImmutableMap::of, + directories, + BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER)) + .put( + BzlmodRepoRuleValue.BZLMOD_REPO_RULE, + new BzlmodRepoRuleFunction(ruleClassProvider, directories)) + .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) + .put( + SkyFunctions.MODULE_FILE, + new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) + .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) + .put( + SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, + new ClientEnvironmentFunction( + new AtomicReference<>(ImmutableMap.of("BZLMOD_ALLOW_YANKED_VERSIONS", "")))) + .put( + updateLockfileFunction, + new SkyFunction() { + @Nullable + @Override + public SkyValue compute(SkyKey skyKey, Environment env) + throws SkyFunctionException, InterruptedException { + + UpdateLockFileKey key = (UpdateLockFileKey) skyKey; + BzlmodFlagsAndEnvVars flags = BazelDepGraphFunction.getFlagsAndEnvVars(env); + if (flags == null) { + return null; + } + + ImmutableMap localOverrideHashes = + BazelDepGraphFunction.getLocalOverridesHashes(key.overrides(), env); + if (localOverrideHashes == null) { + return null; + } + BazelLockFileFunction.updateLockedModule( + rootDirectory, + key.moduleHash(), + flags, + localOverrideHashes, + key.depGraph()); + return new SkyValue() {}; + } + }) + .buildOrThrow(), + differencer); + + PrecomputedValue.STARLARK_SEMANTICS.set( + differencer, + StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); + ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); + ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( + differencer, BazelCompatibilityMode.ERROR); + BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( + differencer, CheckDirectDepsMode.ERROR); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set(differencer, ImmutableMap.of()); + RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING.set( + differencer, RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY); + } + + @Test + public void simpleModule() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')", + "bazel_dep(name = 'dep_1', version = '1.0')", + "bazel_dep(name = 'dep_2', version = '2.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.builder() + .put(ModuleKey.ROOT, rootValue.getModule()) + .buildOrThrow(); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + BazelLockFileValue value = result.get(BazelLockFileValue.KEY); + assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); + } + + @Test + public void moduleWithFlags() throws Exception { + // Test having --override_module, --ignore_dev_dependency, --check_bazel_compatibility + // --check_direct_dependencies & --registry + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.builder() + .put(ModuleKey.ROOT, rootValue.getModule()) + .buildOrThrow(); + + ImmutableList yankedVersions = ImmutableList.of("2.4", "2.3"); + LocalPathOverride override = LocalPathOverride.create("override_path"); + + ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); + ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of("my_dep_1", override)); + + BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, yankedVersions); + BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( + differencer, CheckDirectDepsMode.ERROR); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( + differencer, BazelCompatibilityMode.ERROR); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + BazelLockFileValue value = result.get(BazelLockFileValue.KEY); + assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); + assertThat(value.getFlags().ignoreDevDependency()).isTrue(); + assertThat(value.getFlags().allowedYankedVersions()).isEqualTo(yankedVersions); + assertThat(value.getFlags().directDependenciesMode()) + .isEqualTo(CheckDirectDepsMode.ERROR.toString()); + assertThat(value.getFlags().compatibilityMode()) + .isEqualTo(BazelCompatibilityMode.ERROR.toString()); + } + + @Test + public void moduleWithLocalOverrides() throws IOException, InterruptedException { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='root',version='0.1')", + "local_path_override(module_name='ss',path='code_for_ss')"); + scratch.file( + rootDirectory.getRelative("code_for_ss/MODULE.bazel").getPathString(), + "module(name='ss',version='1.0')"); + scratch.file(rootDirectory.getRelative("code_for_ss/WORKSPACE").getPathString()); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.builder() + .put(ModuleKey.ROOT, rootValue.getModule()) + .buildOrThrow(); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + BazelLockFileValue value = result.get(BazelLockFileValue.KEY); + assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); + assertThat(value.getLocalOverrideHashes()).isNotEmpty(); + } + + @Test + public void fullModule() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "module(name='my_root', version='1.0')", + "register_toolchains('//my:toolchain', '//my:toolchain2')", + "ext1 = use_extension('//:defs.bzl','ext_1')", + "use_repo(ext1,'myrepo')", + "ext2 = use_extension('@ext//:defs.bzl','ext_2')", + "ext2.tag(file='@myrepo//:Hello1.txt')", + "ext2.tag(file='@myrepo//:Hello2.txt')", + "use_repo(ext2,'ext_repo')"); + + EvaluationResult rootResult = + evaluator.evaluate( + ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + if (rootResult.hasError()) { + fail(rootResult.getError().toString()); + } + RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); + + ImmutableMap depGraph = + ImmutableMap.builder() + .put(ModuleKey.ROOT, rootValue.getModule()) + .buildOrThrow(); + + UpdateLockFileKey key = + UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(key), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + + BazelLockFileValue value = result.get(BazelLockFileValue.KEY); + assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); + } + + @AutoValue + abstract static class UpdateLockFileKey implements SkyKey { + + abstract String moduleHash(); + abstract ImmutableMap depGraph(); + + abstract ImmutableMap overrides(); + + static UpdateLockFileKey create( + String moduleHash, + ImmutableMap depGraph, + ImmutableMap overrides) { + return new AutoValue_BazelLockFileFunctionTest_UpdateLockFileKey( + moduleHash, depGraph, overrides); + } + + @Override + public SkyFunctionName functionName() { + return updateLockfileFunction; + } + } +} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 8d77b6622be04d..03fdbacf5e170f 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.pkgcache.PathPackageLocator; @@ -111,7 +112,8 @@ public void setup() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, @@ -128,6 +130,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.OFF); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 37b92c3609dfcc..212bf5a2a2214b 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -29,6 +29,7 @@ import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -123,7 +124,8 @@ public void setup() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( SkyFunctions.MODULE_FILE, @@ -135,6 +137,7 @@ public void setup() throws Exception { differencer); PrecomputedValue.STARLARK_SEMANTICS.set(differencer, StarlarkSemantics.DEFAULT); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of()); ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); @@ -142,6 +145,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); } @Test @@ -174,6 +178,50 @@ public void testRepoSpec_bazelModule() throws Exception { assertThat(repoRule.getAttr("path", Type.STRING)).isEqualTo("/usr/local/modules/ccc~2.0"); } + @Test + public void testRepoSpec_lockfile() throws Exception { + scratch.file( + workspaceRoot.getRelative("MODULE.bazel").getPathString(), + "bazel_dep(name='bbb',version='2.0')"); + + FakeRegistry registry = + registryFactory + .newFakeRegistry("/usr/local/modules") + .addModule(createModuleKey("bbb", "2.0"), "module(name='bbb', version='2.0')"); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); + + RepositoryName repo = RepositoryName.create("bbb~2.0"); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(BzlmodRepoRuleValue.key(repo)), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + BzlmodRepoRuleValue bzlmodRepoRuleValue = result.get(BzlmodRepoRuleValue.key(repo)); + Rule repoRule = bzlmodRepoRuleValue.getRule(); + assertThat(repoRule.getName()).isEqualTo("bbb~2.0"); + + /* Rerun the setup to: + 1.Reset evaluator to remove the sky values cache. + 2.Reset registry factory to be empty (now "bbb" doesn't exist at all) + without the lockfile "bbb" should not be found and this should fail + */ + setup(); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); + registry = registryFactory.newFakeRegistry("/usr/local/modules"); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + + // Calling again should read from lockfile and still find ccc + result = evaluator.evaluate(ImmutableList.of(BzlmodRepoRuleValue.key(repo)), evaluationContext); + if (result.hasError()) { + fail(result.getError().toString()); + } + bzlmodRepoRuleValue = result.get(BzlmodRepoRuleValue.key(repo)); + repoRule = bzlmodRepoRuleValue.getRule(); + assertThat(repoRule.getName()).isEqualTo("bbb~2.0"); + } + @Test public void testRepoSpec_nonRegistryOverride() throws Exception { scratch.file( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodTestUtil.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodTestUtil.java index 2af63ead6d9d35..484e40e0dce23e 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodTestUtil.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodTestUtil.java @@ -305,7 +305,7 @@ public Tag build() { return Tag.builder() .setTagName(tagName) .setLocation(Location.BUILTIN) - .setAttributeValues(attrValuesBuilder.buildImmutable()) + .setAttributeValues(AttributeValues.create(attrValuesBuilder.buildImmutable())) .setDevDependency(devDependency) .build(); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java index e731065e4b4304..593a3c5484da7a 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java @@ -73,8 +73,9 @@ public RepoSpec getRepoSpec( return RepoSpec.builder() .setRuleClassName("local_repository") .setAttributes( - ImmutableMap.of( - "name", repoName.getName(), "path", rootPath + "/" + repoName.getName())) + AttributeValues.create( + ImmutableMap.of( + "name", repoName.getName(), "path", rootPath + "/" + repoName.getName()))) .build(); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index d9ae5eff654b8e..d18e81248c2afc 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -201,7 +201,9 @@ public void testGetLocalPathRepoSpec() throws Exception { .isEqualTo( RepoSpec.builder() .setRuleClassName("local_repository") - .setAttributes(ImmutableMap.of("name", "foorepo", "path", "/hello/bar/project_x")) + .setAttributes( + AttributeValues.create( + ImmutableMap.of("name", "foorepo", "path", "/hello/bar/project_x"))) .build()); } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 69b3adba649dc4..c4ac3a67d72a88 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -31,6 +31,7 @@ import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryFunction; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; @@ -247,7 +248,8 @@ public void setup() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()) .put(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) @@ -278,6 +280,7 @@ public void setup() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); // Set up a simple repo rule. registry.addModule( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java index bbdeb0d4d4d33a..ecd8353f8b4605 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java @@ -481,9 +481,10 @@ public void testModuleExtensions_good() throws Exception { Tag.builder() .setTagName("tag") .setAttributeValues( - Dict.builder() - .put("key", "val") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("key", "val") + .buildImmutable())) .setDevDependency(false) .setLocation( Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 4, 11)) @@ -499,9 +500,10 @@ public void testModuleExtensions_good() throws Exception { Tag.builder() .setTagName("tag1") .setAttributeValues( - Dict.builder() - .put("key1", "val1") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("key1", "val1") + .buildImmutable())) .setDevDependency(false) .setLocation( Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 7, 12)) @@ -510,9 +512,10 @@ public void testModuleExtensions_good() throws Exception { Tag.builder() .setTagName("tag2") .setAttributeValues( - Dict.builder() - .put("key2", "val2") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("key2", "val2") + .buildImmutable())) .setDevDependency(false) .setLocation( Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 8, 12)) @@ -529,9 +532,10 @@ public void testModuleExtensions_good() throws Exception { Tag.builder() .setTagName("dep") .setAttributeValues( - Dict.builder() - .put("coord", "junit") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("coord", "junit") + .buildImmutable())) .setDevDependency(false) .setLocation( Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 12, 10)) @@ -540,9 +544,10 @@ public void testModuleExtensions_good() throws Exception { Tag.builder() .setTagName("dep") .setAttributeValues( - Dict.builder() - .put("coord", "guava") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("coord", "guava") + .buildImmutable())) .setDevDependency(false) .setLocation( Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 14, 10)) @@ -593,9 +598,10 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { Tag.builder() .setTagName("tag") .setAttributeValues( - Dict.builder() - .put("name", "tag1") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("name", "tag1") + .buildImmutable())) .setDevDependency(true) .setLocation( Location.fromFileLineColumn("/MODULE.bazel", 2, 11)) @@ -604,9 +610,10 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { Tag.builder() .setTagName("tag") .setAttributeValues( - Dict.builder() - .put("name", "tag2") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("name", "tag2") + .buildImmutable())) .setDevDependency(false) .setLocation( Location.fromFileLineColumn("/MODULE.bazel", 5, 11)) @@ -615,9 +622,10 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { Tag.builder() .setTagName("tag") .setAttributeValues( - Dict.builder() - .put("name", "tag3") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("name", "tag3") + .buildImmutable())) .setDevDependency(true) .setLocation( Location.fromFileLineColumn("/MODULE.bazel", 8, 11)) @@ -626,9 +634,10 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { Tag.builder() .setTagName("tag") .setAttributeValues( - Dict.builder() - .put("name", "tag4") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("name", "tag4") + .buildImmutable())) .setDevDependency(false) .setLocation( Location.fromFileLineColumn("/MODULE.bazel", 11, 11)) @@ -680,9 +689,10 @@ public void testModuleExtensions_duplicateProxy_asDep() throws Exception { Tag.builder() .setTagName("tag") .setAttributeValues( - Dict.builder() - .put("name", "tag2") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("name", "tag2") + .buildImmutable())) .setDevDependency(false) .setLocation( Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 6, 11)) @@ -691,9 +701,10 @@ public void testModuleExtensions_duplicateProxy_asDep() throws Exception { Tag.builder() .setTagName("tag") .setAttributeValues( - Dict.builder() - .put("name", "tag4") - .buildImmutable()) + AttributeValues.create( + Dict.builder() + .put("name", "tag4") + .buildImmutable())) .setDevDependency(false) .setLocation( Location.fromFileLineColumn("mymod@1.0/MODULE.bazel", 12, 11)) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecTest.java index 754f65d29c1c8a..e7a84d633a3b02 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecTest.java @@ -15,9 +15,9 @@ package com.google.devtools.build.lib.bazel.bzlmod; import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth8.assertThat; import com.google.common.collect.ImmutableMap; +import java.util.Map; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -31,12 +31,12 @@ public void nativeRepoSpecTest() { RepoSpec repoSpec = RepoSpec.builder() .setRuleClassName("local_repository") - .setAttributes(ImmutableMap.of("path", "/foo/bar")) + .setAttributes(AttributeValues.create(ImmutableMap.of("path", "/foo/bar"))) .build(); assertThat(repoSpec.isNativeRepoRule()).isTrue(); assertThat(repoSpec.ruleClassName()).isEqualTo("local_repository"); assertThat(repoSpec.getRuleClass()).isEqualTo("local_repository"); - assertThat(repoSpec.attributes()).containsExactly("path", "/foo/bar"); + assertThat((Map) repoSpec.attributes().attributes()).containsExactly("path", "/foo/bar"); } @Test @@ -45,12 +45,13 @@ public void starlarkRepoSpecTest() { RepoSpec.builder() .setBzlFile("//pkg:repo.bzl") .setRuleClassName("my_repo") - .setAttributes(ImmutableMap.of("attr1", "foo", "attr2", "bar")) + .setAttributes(AttributeValues.create(ImmutableMap.of("attr1", "foo", "attr2", "bar"))) .build(); assertThat(repoSpec.isNativeRepoRule()).isFalse(); - assertThat(repoSpec.bzlFile()).hasValue("//pkg:repo.bzl"); + assertThat(repoSpec.bzlFile()).isEqualTo("//pkg:repo.bzl"); assertThat(repoSpec.ruleClassName()).isEqualTo("my_repo"); assertThat(repoSpec.getRuleClass()).isEqualTo("//pkg:repo.bzl%my_repo"); - assertThat(repoSpec.attributes()).containsExactly("attr1", "foo", "attr2", "bar"); + assertThat((Map) repoSpec.attributes().attributes()) + .containsExactly("attr1", "foo", "attr2", "bar"); } } diff --git a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java index c791adae7d933c..6e705ff8b87833 100644 --- a/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java +++ b/src/test/java/com/google/devtools/build/lib/query2/testutil/SkyframeQueryHelper.java @@ -25,12 +25,14 @@ import com.google.devtools.build.lib.analysis.ServerDirectories; import com.google.devtools.build.lib.analysis.config.BuildOptions; import com.google.devtools.build.lib.analysis.util.AnalysisMock; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -99,7 +101,6 @@ public abstract class SkyframeQueryHelper extends AbstractQueryHelper { private FakeRegistry registry; protected Path rootDirectory; - protected Path installBase; protected Path outputBase; protected Path moduleRoot; protected BlazeDirectories directories; @@ -369,7 +370,9 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, - BazelCompatibilityMode.ERROR))) + BazelCompatibilityMode.ERROR), + PrecomputedValue.injected( + BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF))) .setEnvironmentExtensions(getEnvironmentExtensions()) .build(ruleClassProvider, fileSystem); SkyframeExecutor skyframeExecutor = @@ -412,6 +415,7 @@ protected SkyframeExecutor createSkyframeExecutor(ConfiguredRuleClassProvider ru PrecomputedValue.injected( BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)) + .add(PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)) .build()); SkyframeExecutorTestHelper.process(skyframeExecutor); return skyframeExecutor; diff --git a/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java b/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java index 78fb2d9ad04678..39e7aeafc023d7 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/LabelBuildSettingTest.java @@ -21,11 +21,13 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Injected; @@ -57,7 +59,8 @@ protected ImmutableList extraPrecomputedValues() { PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); } private void writeRulesBzl(String type) throws Exception { diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index 0f359f95a1c11b..9f05f91f32bd8a 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -31,12 +31,14 @@ import com.google.devtools.build.lib.analysis.ServerDirectories; import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryFunction; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; @@ -225,7 +227,7 @@ public void setupDelegator() throws Exception { .put( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootPath, ImmutableMap.of())) - .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) + .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, @@ -257,6 +259,7 @@ public void setupDelegator() throws Exception { differencer, CheckDirectDepsMode.WARNING); BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( differencer, BazelCompatibilityMode.ERROR); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.OFF); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java index c13c30d90bf4f9..61230364befbd8 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/BzlLoadFunctionTest.java @@ -22,11 +22,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.cmdline.BazelModuleContext; import com.google.devtools.build.lib.cmdline.Label; @@ -110,7 +112,8 @@ protected ImmutableList extraPrecomputedValues() { PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); } @Before diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java index aefc7e6871233a..2d00ff7c3ed816 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/PrepareDepsOfPatternsFunctionTest.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.eventbus.EventBus; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; @@ -28,6 +29,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.Version; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.packages.NoSuchPackageException; @@ -244,7 +246,8 @@ protected ImmutableList extraPrecomputedValues() { PrecomputedValue.injected( BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, ImmutableList.of()), PrecomputedValue.injected( - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); } // Helpers: diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java index 1a3547bcd6a649..f44837d61141d6 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredExecutionPlatformsFunctionTest.java @@ -25,11 +25,13 @@ import com.google.common.truth.IterableSubject; import com.google.devtools.build.lib.analysis.ViewCreationFailedException; import com.google.devtools.build.lib.analysis.platform.ConstraintCollection; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.rules.platform.ToolchainTestCase; @@ -110,7 +112,8 @@ protected ImmutableList extraPrecomputedValues() { PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java index 31785091ffd53e..5adfeb0b2f4660 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RegisteredToolchainsFunctionTest.java @@ -23,11 +23,13 @@ import com.google.common.testing.EqualsTester; import com.google.devtools.build.lib.analysis.platform.DeclaredToolchainInfo; import com.google.devtools.build.lib.analysis.platform.ToolchainTypeInfo; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.rules.platform.ToolchainTestCase; @@ -66,7 +68,8 @@ protected ImmutableList extraPrecomputedValues() { PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java b/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java index 3aa18259bdb461..06672b18ddc308 100644 --- a/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/skyframe/RepositoryMappingFunctionTest.java @@ -26,12 +26,14 @@ import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.Version; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.RepositoryMapping; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.packages.NoSuchPackageException; @@ -85,7 +87,8 @@ protected ImmutableList extraPrecomputedValues() thro PrecomputedValue.injected( BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, CheckDirectDepsMode.WARNING), PrecomputedValue.injected( - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR)); + BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, BazelCompatibilityMode.ERROR), + PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, LockfileMode.OFF)); } @Override diff --git a/src/test/py/bazel/BUILD b/src/test/py/bazel/BUILD index be95008b2e1343..4fa55fe5dd2a5d 100644 --- a/src/test/py/bazel/BUILD +++ b/src/test/py/bazel/BUILD @@ -282,6 +282,58 @@ py_test( ], ) +py_test( + name = "bazel_lockfile_test", + size = "medium", + srcs = ["bzlmod/bazel_lockfile_test.py"], + tags = [ + "requires-network", + ], + deps = [ + ":bzlmod_test_utils", + ":test_base", + ], +) + +py_test( + name = "bazel_overrides_test", + size = "medium", + srcs = ["bzlmod/bazel_overrides_test.py"], + tags = [ + "requires-network", + ], + deps = [ + ":bzlmod_test_utils", + ":test_base", + ], +) + +py_test( + name = "bazel_repo_mapping_test", + size = "medium", + srcs = ["bzlmod/bazel_repo_mapping_test.py"], + tags = [ + "requires-network", + ], + deps = [ + ":bzlmod_test_utils", + ":test_base", + ], +) + +py_test( + name = "bazel_yanked_versions_test", + size = "medium", + srcs = ["bzlmod/bazel_yanked_versions_test.py"], + tags = [ + "requires-network", + ], + deps = [ + ":bzlmod_test_utils", + ":test_base", + ], +) + py_test( name = "starlark_options_test", srcs = ["starlark_options_test.py"], diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py new file mode 100644 index 00000000000000..3568ea699e7863 --- /dev/null +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -0,0 +1,263 @@ +# pylint: disable=g-backslash-continuation +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=g-long-ternary + +import os +import tempfile +import unittest + +from src.test.py.bazel import test_base +from src.test.py.bazel.bzlmod.test_utils import BazelRegistry +from src.test.py.bazel.bzlmod.test_utils import scratchFile + + +class BazelLockfileTest(test_base.TestBase): + + def setUp(self): + test_base.TestBase.setUp(self) + self.registries_work_dir = tempfile.mkdtemp(dir=self._test_cwd) + self.main_registry = BazelRegistry( + os.path.join(self.registries_work_dir, 'main') + ) + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'aaa', '1.1' + ).createCcModule('bbb', '1.0', {'aaa': '1.0'}).createCcModule( + 'bbb', '1.1', {'aaa': '1.1'} + ).createCcModule( + 'ccc', '1.1', {'aaa': '1.1', 'bbb': '1.1'} + ) + self.ScratchFile( + '.bazelrc', + [ + # In ipv6 only network, this has to be enabled. + # 'startup --host_jvm_args=-Djava.net.preferIPv6Addresses=true', + 'common --enable_bzlmod', + 'common --registry=' + self.main_registry.getURL(), + # We need to have BCR here to make sure built-in modules like + # bazel_tools can work. + 'common --registry=https://bcr.bazel.build', + 'common --verbose_failures', + # Set an explicit Java language version + 'common --java_language_version=8', + 'common --tool_java_language_version=8', + 'common --lockfile_mode=update', + ], + ) + self.ScratchFile('WORKSPACE') + # The existence of WORKSPACE.bzlmod prevents WORKSPACE prefixes or suffixes + # from being used; this allows us to test built-in modules actually work + self.ScratchFile('WORKSPACE.bzlmod') + + def testChangeModuleInRegistryWithoutLockfile(self): + # Add module 'sss' to the registry with dep on 'aaa' + self.main_registry.createCcModule('sss', '1.3', {'aaa': '1.1'}) + # Create a project with deps on 'sss' + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "sss", version = "1.3")', + ], + ) + self.ScratchFile('BUILD', ['filegroup(name = "hello")']) + self.RunBazel( + [ + 'build', + '--nobuild', + '--lockfile_mode=off', + '//:all', + ], + allow_failure=False, + ) + + # Change registry -> update 'sss' module file (corrupt it) + module_dir = self.main_registry.root.joinpath('modules', 'sss', '1.3') + scratchFile(module_dir.joinpath('MODULE.bazel'), ['whatever!']) + + # Clean bazel to empty any cache of the deps tree + self.RunBazel(['clean', '--expunge']) + # Runing again will try to get 'sss' which should produce an error + exit_code, _, stderr = self.RunBazel( + [ + 'build', + '--nobuild', + '--lockfile_mode=off', + '//:all', + ], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + ( + 'ERROR: Error computing the main repository mapping: error parsing' + ' MODULE.bazel file for sss@1.3' + ), + stderr, + ) + + def testChangeModuleInRegistryWithLockfile(self): + # Add module 'sss' to the registry with dep on 'aaa' + self.main_registry.createCcModule('sss', '1.3', {'aaa': '1.1'}) + # Create a project with deps on 'sss' + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "sss", version = "1.3")', + ], + ) + self.ScratchFile('BUILD', ['filegroup(name = "hello")']) + self.RunBazel( + [ + 'build', + '--nobuild', + '//:all', + ], + allow_failure=False, + ) + + # Change registry -> update 'sss' module file (corrupt it) + module_dir = self.main_registry.root.joinpath('modules', 'sss', '1.3') + scratchFile(module_dir.joinpath('MODULE.bazel'), ['whatever!']) + + # Clean bazel to empty any cache of the deps tree + self.RunBazel(['clean', '--expunge']) + # Running with the lockfile, should not recognize the registry changes + # hence find no errors + self.RunBazel(['build', '--nobuild', '//:all'], allow_failure=False) + + def testChangeFlagWithLockfile(self): + # Add module 'sss' to the registry with dep on 'aaa' + self.main_registry.createCcModule('sss', '1.3', {'aaa': '1.1'}) + # Create a project with deps on 'sss' + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "sss", version = "1.3")', + ], + ) + self.ScratchFile('BUILD', ['filegroup(name = "hello")']) + self.RunBazel( + ['build', '--nobuild', '//:all'], + allow_failure=False, + ) + + # Change registry -> update 'sss' module file (corrupt it) + module_dir = self.main_registry.root.joinpath('modules', 'sss', '1.3') + scratchFile(module_dir.joinpath('MODULE.bazel'), ['whatever!']) + + # Clean bazel to empty any cache of the deps tree + self.RunBazel(['clean', '--expunge']) + # Running with the lockfile, but adding a flag should cause resolution rerun + exit_code, _, stderr = self.RunBazel( + [ + 'build', + '--nobuild', + '--check_direct_dependencies=error', + '//:all', + ], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + "ERROR: sss@1.3/MODULE.bazel:1:9: invalid character: '!'", stderr + ) + + def testLockfileErrorMode(self): + self.ScratchFile('MODULE.bazel', []) + self.ScratchFile('BUILD', ['filegroup(name = "hello")']) + self.RunBazel( + [ + 'build', + '--nobuild', + '--check_direct_dependencies=warning', + '//:all', + ], + allow_failure=False, + ) + + # Run with updated module and a different flag + self.ScratchFile('MODULE.bazel', ['module(name="lala")']) + exit_code, _, stderr = self.RunBazel( + [ + 'build', + '--nobuild', + '--check_direct_dependencies=error', + '--lockfile_mode=error', + '//:all', + ], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + ( + 'ERROR: Error computing the main repository mapping: Lock file is' + ' no longer up-to-date because: the root MODULE.bazel has been' + ' modified, the value of --check_direct_dependencies flag has' + ' been modified' + ), + stderr, + ) + + def testLocalOverrideWithErrorMode(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'module(name="lala")', + 'bazel_dep(name="bar")', + 'local_path_override(module_name="bar",path="bar")', + ], + ) + self.ScratchFile('BUILD', ['filegroup(name = "hello")']) + self.ScratchFile('bar/MODULE.bazel', ['module(name="bar")']) + self.ScratchFile('bar/WORKSPACE', []) + self.ScratchFile('bar/BUILD', ['filegroup(name = "hello from bar")']) + self.RunBazel( + [ + 'build', + '--nobuild', + '//:all', + ], + allow_failure=False, + ) + + # Run with updated module and a different flag + self.ScratchFile( + 'bar/MODULE.bazel', + [ + 'module(name="bar")', + 'bazel_dep(name="hmmm")', + ], + ) + exit_code, _, stderr = self.RunBazel( + [ + 'build', + '--nobuild', + '--lockfile_mode=error', + '//:all', + ], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + ( + 'ERROR: Error computing the main repository mapping: Lock file is' + ' no longer up-to-date because: The MODULE.bazel file has changed' + ' for the overriden module: bar' + ), + stderr, + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/test/py/bazel/bzlmod/bazel_module_test.py b/src/test/py/bazel/bzlmod/bazel_module_test.py index 8d6747a37b4912..941823f11fb680 100644 --- a/src/test/py/bazel/bzlmod/bazel_module_test.py +++ b/src/test/py/bazel/bzlmod/bazel_module_test.py @@ -31,24 +31,15 @@ def setUp(self): self.registries_work_dir = tempfile.mkdtemp(dir=self._test_cwd) self.main_registry = BazelRegistry( os.path.join(self.registries_work_dir, 'main')) - self.main_registry.createCcModule('aaa', '1.0') \ - .createCcModule('aaa', '1.1') \ - .createCcModule('bbb', '1.0', {'aaa': '1.0'}, {'aaa': 'com_foo_aaa'}) \ - .createCcModule('bbb', '1.1', {'aaa': '1.1'}) \ - .createCcModule('ccc', '1.1', {'aaa': '1.1', 'bbb': '1.1'}) \ - .createCcModule('ddd', '1.0', {'yanked1': '1.0', 'yanked2': '1.0'}) \ - .createCcModule('eee', '1.0', {'yanked1': '1.0'}) \ - .createCcModule('yanked1', '1.0') \ - .createCcModule('yanked2', '1.0') \ - .addMetadata('yanked1', yanked_versions={'1.0': 'dodgy'}) \ - .addMetadata('yanked2', yanked_versions={'1.0': 'sketchy'}) - self.writeBazelrcFile() - self.ScratchFile('WORKSPACE') - # The existence of WORKSPACE.bzlmod prevents WORKSPACE prefixes or suffixes - # from being used; this allows us to test built-in modules actually work - self.ScratchFile('WORKSPACE.bzlmod') - - def writeBazelrcFile(self, allow_yanked_versions=True): + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'aaa', '1.1' + ).createCcModule( + 'bbb', '1.0', {'aaa': '1.0'}, {'aaa': 'com_foo_aaa'} + ).createCcModule( + 'bbb', '1.1', {'aaa': '1.1'} + ).createCcModule( + 'ccc', '1.1', {'aaa': '1.1', 'bbb': '1.1'} + ) self.ScratchFile( '.bazelrc', [ @@ -60,10 +51,16 @@ def writeBazelrcFile(self, allow_yanked_versions=True): # bazel_tools can work. 'common --registry=https://bcr.bazel.build', 'common --verbose_failures', - ] + ([ - # Disable yanked version check so we are not affected BCR changes. - 'common --allow_yanked_versions=all', - ] if allow_yanked_versions else [])) + # Set an explicit Java language version + 'common --java_language_version=8', + 'common --tool_java_language_version=8', + 'common --lockfile_mode=update', + ], + ) + self.ScratchFile('WORKSPACE') + # The existence of WORKSPACE.bzlmod prevents WORKSPACE prefixes or suffixes + # from being used; this allows us to test built-in modules actually work + self.ScratchFile('WORKSPACE.bzlmod') def writeMainProjectFiles(self): self.ScratchFile('aaa.patch', [ @@ -152,115 +149,6 @@ def testSimpleDiamond(self): self.assertIn('main function => bbb@1.0', stdout) self.assertIn('bbb@1.0 => aaa@1.1', stdout) - def testSingleVersionOverrideWithPatch(self): - self.writeMainProjectFiles() - self.ScratchFile( - 'MODULE.bazel', - [ - 'bazel_dep(name = "aaa", version = "1.1")', - 'bazel_dep(name = "bbb", version = "1.1")', - # Both main and bbb@1.1 has to depend on the locally patched aaa@1.0 - 'single_version_override(', - ' module_name = "aaa",', - ' version = "1.0",', - ' patches = ["//:aaa.patch"],', - ' patch_strip = 1,', - ')', - ]) - _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) - self.assertIn('main function => aaa@1.0 (locally patched)', stdout) - self.assertIn('main function => bbb@1.1', stdout) - self.assertIn('bbb@1.1 => aaa@1.0 (locally patched)', stdout) - - def testRegistryOverride(self): - self.writeMainProjectFiles() - another_registry = BazelRegistry( - os.path.join(self.registries_work_dir, 'another'), - ' from another registry') - another_registry.createCcModule('aaa', '1.0') - self.ScratchFile('MODULE.bazel', [ - 'bazel_dep(name = "aaa", version = "1.0")', - 'bazel_dep(name = "bbb", version = "1.0")', - 'single_version_override(', - ' module_name = "aaa",', - ' registry = "%s",' % another_registry.getURL(), - ')', - ]) - _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) - self.assertIn('main function => aaa@1.0 from another registry', stdout) - self.assertIn('main function => bbb@1.0', stdout) - self.assertIn('bbb@1.0 => aaa@1.0 from another registry', stdout) - - def testArchiveOverride(self): - self.writeMainProjectFiles() - archive_aaa_1_0 = self.main_registry.archives.joinpath('aaa.1.0.zip') - self.ScratchFile('MODULE.bazel', [ - 'bazel_dep(name = "aaa", version = "1.1")', - 'bazel_dep(name = "bbb", version = "1.1")', - 'archive_override(', - ' module_name = "aaa",', - ' urls = ["%s"],' % archive_aaa_1_0.as_uri(), - ' patches = ["//:aaa.patch"],', - ' patch_strip = 1,', - ')', - ]) - _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) - self.assertIn('main function => aaa@1.0 (locally patched)', stdout) - self.assertIn('main function => bbb@1.1', stdout) - self.assertIn('bbb@1.1 => aaa@1.0 (locally patched)', stdout) - - def testGitOverride(self): - self.writeMainProjectFiles() - - src_aaa_1_0 = self.main_registry.projects.joinpath('aaa', '1.0') - self.RunProgram(['git', 'init'], cwd=src_aaa_1_0, allow_failure=False) - self.RunProgram(['git', 'config', 'user.name', 'tester'], - cwd=src_aaa_1_0, - allow_failure=False) - self.RunProgram(['git', 'config', 'user.email', 'tester@foo.com'], - cwd=src_aaa_1_0, - allow_failure=False) - self.RunProgram(['git', 'add', './'], cwd=src_aaa_1_0, allow_failure=False) - self.RunProgram(['git', 'commit', '-m', 'Initial commit.'], - cwd=src_aaa_1_0, - allow_failure=False) - _, stdout, _ = self.RunProgram(['git', 'rev-parse', 'HEAD'], - cwd=src_aaa_1_0, - allow_failure=False) - commit = stdout[0].strip() - - self.ScratchFile('MODULE.bazel', [ - 'bazel_dep(name = "aaa", version = "1.1")', - 'bazel_dep(name = "bbb", version = "1.1")', - 'git_override(', - ' module_name = "aaa",', - ' remote = "%s",' % src_aaa_1_0.as_uri(), - ' commit = "%s",' % commit, - ' patches = ["//:aaa.patch"],', - ' patch_strip = 1,', - ')', - ]) - _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) - self.assertIn('main function => aaa@1.0 (locally patched)', stdout) - self.assertIn('main function => bbb@1.1', stdout) - self.assertIn('bbb@1.1 => aaa@1.0 (locally patched)', stdout) - - def testLocalPathOverride(self): - src_aaa_1_0 = self.main_registry.projects.joinpath('aaa', '1.0') - self.writeMainProjectFiles() - self.ScratchFile('MODULE.bazel', [ - 'bazel_dep(name = "aaa", version = "1.1")', - 'bazel_dep(name = "bbb", version = "1.1")', - 'local_path_override(', - ' module_name = "aaa",', - ' path = "%s",' % str(src_aaa_1_0.resolve()).replace('\\', '/'), - ')', - ]) - _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) - self.assertIn('main function => aaa@1.0', stdout) - self.assertIn('main function => bbb@1.1', stdout) - self.assertIn('bbb@1.1 => aaa@1.0', stdout) - def testRemotePatchForBazelDep(self): patch_file = self.ScratchFile('aaa.patch', [ '--- a/aaa.cc', @@ -379,121 +267,6 @@ def testRepositoryRuleErrorInModuleExtensionFailsTheBuild(self): in line for line in stderr ])) - def testCmdAbsoluteModuleOverride(self): - # test commandline_overrides takes precedence over local_path_override - self.ScratchFile('MODULE.bazel', [ - 'bazel_dep(name = "ss", version = "1.0")', - 'local_path_override(', - ' module_name = "ss",', - ' path = "%s",' % self.Path('aa'), - ')', - ]) - self.ScratchFile('BUILD') - self.ScratchFile('WORKSPACE') - - self.ScratchFile('aa/MODULE.bazel', [ - 'module(name=\'ss\')', - ]) - self.ScratchFile('aa/BUILD', [ - 'filegroup(name = "never_ever")', - ]) - self.ScratchFile('aa/WORKSPACE') - - self.ScratchFile('bb/MODULE.bazel', [ - 'module(name=\'ss\')', - ]) - self.ScratchFile('bb/BUILD', [ - 'filegroup(name = "choose_me")', - ]) - self.ScratchFile('bb/WORKSPACE') - - _, _, stderr = self.RunBazel( - ['build', '@ss//:all', '--override_module', 'ss=' + self.Path('bb')], - allow_failure=False, - ) - # module file override should be ignored, and bb directory should be used - self.assertIn( - 'Target @ss~override//:choose_me up-to-date (nothing to build)', stderr) - - def testCmdRelativeModuleOverride(self): - self.ScratchFile('aa/WORKSPACE') - self.ScratchFile( - 'aa/MODULE.bazel', - [ - 'bazel_dep(name = "ss", version = "1.0")', - ], - ) - self.ScratchFile('aa/BUILD') - - self.ScratchFile('aa/cc/BUILD') - - self.ScratchFile('bb/WORKSPACE') - self.ScratchFile( - 'bb/MODULE.bazel', - [ - 'module(name="ss")', - ], - ) - self.ScratchFile( - 'bb/BUILD', - [ - 'filegroup(name = "choose_me")', - ], - ) - - _, _, stderr = self.RunBazel( - [ - 'build', - '@ss//:all', - '--override_module', - 'ss=../../bb', - '--enable_bzlmod', - ], - cwd=self.Path('aa/cc'), - allow_failure=False, - ) - self.assertIn( - 'Target @ss~override//:choose_me up-to-date (nothing to build)', stderr - ) - - def testCmdWorkspaceRelativeModuleOverride(self): - self.ScratchFile('WORKSPACE') - self.ScratchFile( - 'MODULE.bazel', - [ - 'bazel_dep(name = "ss", version = "1.0")', - ], - ) - self.ScratchFile('BUILD') - self.ScratchFile('aa/BUILD') - self.ScratchFile('bb/WORKSPACE') - self.ScratchFile( - 'bb/MODULE.bazel', - [ - 'module(name="ss")', - ], - ) - self.ScratchFile( - 'bb/BUILD', - [ - 'filegroup(name = "choose_me")', - ], - ) - - _, _, stderr = self.RunBazel( - [ - 'build', - '@ss//:all', - '--override_module', - 'ss=%workspace%/bb', - ], - cwd=self.Path('aa'), - allow_failure=False, - ) - self.assertIn( - 'Target @ss~override//:choose_me up-to-date (nothing to build)', stderr - ) - def testDownload(self): data_path = self.ScratchFile('data.txt', ['some data']) data_url = pathlib.Path(data_path).resolve().as_uri() @@ -519,110 +292,6 @@ def testDownload(self): ]) self.RunBazel(['build', '@no_op//:no_op'], allow_failure=False) - def testNonRegistryOverriddenModulesIgnoreYanked(self): - self.writeBazelrcFile(allow_yanked_versions=False) - src_yanked1 = self.main_registry.projects.joinpath('yanked1', '1.0') - self.ScratchFile('MODULE.bazel', [ - 'bazel_dep(name = "yanked1", version = "1.0")', 'local_path_override(', - ' module_name = "yanked1",', - ' path = "%s",' % str(src_yanked1.resolve()).replace('\\', '/'), ')' - ]) - self.ScratchFile('WORKSPACE') - self.ScratchFile('BUILD', [ - 'cc_binary(', - ' name = "main",', - ' srcs = ["main.cc"],', - ' deps = ["@yanked1//:lib_yanked1"],', - ')', - ]) - self.RunBazel(['build', '--nobuild', '//:main'], allow_failure=False) - - def testContainingYankedDepFails(self): - self.writeBazelrcFile(allow_yanked_versions=False) - self.ScratchFile('MODULE.bazel', [ - 'bazel_dep(name = "yanked1", version = "1.0")', - ]) - self.ScratchFile('WORKSPACE') - self.ScratchFile('BUILD', [ - 'cc_binary(', - ' name = "main",', - ' srcs = ["main.cc"],', - ' deps = ["@ddd//:lib_ddd"],', - ')', - ]) - exit_code, _, stderr = self.RunBazel(['build', '--nobuild', '//:main'], - allow_failure=True) - self.AssertExitCode(exit_code, 48, stderr) - self.assertIn( - 'Yanked version detected in your resolved dependency graph: ' + - 'yanked1@1.0, for the reason: dodgy.', ''.join(stderr)) - - def testAllowedYankedDepsSuccessByFlag(self): - self.writeBazelrcFile(allow_yanked_versions=False) - self.ScratchFile('MODULE.bazel', [ - 'bazel_dep(name = "ddd", version = "1.0")', - ]) - self.ScratchFile('WORKSPACE') - self.ScratchFile('BUILD', [ - 'cc_binary(', - ' name = "main",', - ' srcs = ["main.cc"],', - ' deps = ["@ddd//:lib_ddd"],', - ')', - ]) - self.RunBazel([ - 'build', '--nobuild', '--allow_yanked_versions=yanked1@1.0,yanked2@1.0', - '//:main' - ], - allow_failure=False) - - def testAllowedYankedDepsByEnvVar(self): - self.writeBazelrcFile(allow_yanked_versions=False) - self.ScratchFile('MODULE.bazel', [ - 'bazel_dep(name = "ddd", version = "1.0")', - ]) - self.ScratchFile('WORKSPACE') - self.ScratchFile('BUILD', [ - 'cc_binary(', - ' name = "main",', - ' srcs = ["main.cc"],', - ' deps = ["@ddd//:lib_ddd"],', - ')', - ]) - self.RunBazel( - ['build', '--nobuild', '//:main'], - env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked1@1.0,yanked2@1.0'}, - allow_failure=False) - - # Test changing the env var, the build should fail again. - exit_code, _, stderr = self.RunBazel( - ['build', '--nobuild', '//:main'], - env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked2@1.0'}, - allow_failure=True) - self.AssertExitCode(exit_code, 48, stderr) - self.assertIn( - 'Yanked version detected in your resolved dependency graph: ' + - 'yanked1@1.0, for the reason: dodgy.', ''.join(stderr)) - - def testAllowedYankedDepsSuccessMix(self): - self.writeBazelrcFile(allow_yanked_versions=False) - self.ScratchFile('MODULE.bazel', [ - 'bazel_dep(name = "ddd", version = "1.0")', - ]) - self.ScratchFile('WORKSPACE') - self.ScratchFile('BUILD', [ - 'cc_binary(', - ' name = "main",', - ' srcs = ["main.cc"],', - ' deps = ["@ddd//:lib_ddd"],', - ')', - ]) - self.RunBazel([ - 'build', '--nobuild', '--allow_yanked_versions=yanked1@1.0', '//:main' - ], - env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked2@1.0'}, - allow_failure=False) - def setUpProjectWithLocalRegistryModule(self, dep_name, dep_version): self.main_registry.generateCcSource(dep_name, dep_version) self.main_registry.createLocalPathModule(dep_name, dep_version, @@ -658,301 +327,6 @@ def testLocalRepoInSourceJsonRelativeBasePath(self): _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) self.assertIn('main function => sss@1.3', stdout) - def testRunfilesRepoMappingManifest(self): - self.main_registry.setModuleBasePath('projects') - projects_dir = self.main_registry.projects - - # Set up a "bare_rule" module that contains the "bare_test" rule which - # passes runfiles along - self.main_registry.createLocalPathModule('bare_rule', '1.0', 'bare_rule') - projects_dir.joinpath('bare_rule').mkdir(exist_ok=True) - scratchFile(projects_dir.joinpath('bare_rule', 'WORKSPACE')) - scratchFile(projects_dir.joinpath('bare_rule', 'BUILD')) - # The working directory of a test is the subdirectory of the runfiles - # directory corresponding to the main repository. - scratchFile( - projects_dir.joinpath('bare_rule', 'defs.bzl'), [ - 'def _bare_test_impl(ctx):', - ' exe = ctx.actions.declare_file(ctx.label.name)', - ' ctx.actions.write(exe,', - ' "#/bin/bash\\nif [[ ! -f ../_repo_mapping || ! -s ../_repo_mapping ]]; then\\necho >&2 \\"ERROR: cannot find repo mapping manifest file\\"\\nexit 1\\nfi",', - ' True)', - ' runfiles = ctx.runfiles(files=ctx.files.data)', - ' for data in ctx.attr.data:', - ' runfiles = runfiles.merge(data[DefaultInfo].default_runfiles)', - ' return DefaultInfo(files=depset(direct=[exe]), executable=exe, runfiles=runfiles)', - 'bare_test=rule(', - ' implementation=_bare_test_impl,', - ' attrs={"data":attr.label_list(allow_files=True)},', - ' test=True,', - ')', - ]) - - # Now set up a project tree shaped like a diamond - self.ScratchFile('MODULE.bazel', [ - 'module(name="me",version="1.0")', - 'bazel_dep(name="foo",version="1.0")', - 'bazel_dep(name="bar",version="2.0")', - 'bazel_dep(name="bare_rule",version="1.0")', - ]) - self.ScratchFile('WORKSPACE') - self.ScratchFile('WORKSPACE.bzlmod', ['workspace(name="me_ws")']) - self.ScratchFile('BUILD', [ - 'load("@bare_rule//:defs.bzl", "bare_test")', - 'bare_test(name="me",data=["@foo"])', - ]) - self.main_registry.createLocalPathModule('foo', '1.0', 'foo', { - 'quux': '1.0', - 'bare_rule': '1.0' - }) - self.main_registry.createLocalPathModule('bar', '2.0', 'bar', { - 'quux': '2.0', - 'bare_rule': '1.0' - }) - self.main_registry.createLocalPathModule('quux', '1.0', 'quux1', - {'bare_rule': '1.0'}) - self.main_registry.createLocalPathModule('quux', '2.0', 'quux2', - {'bare_rule': '1.0'}) - for dir_name, build_file in [ - ('foo', 'bare_test(name="foo",data=["@quux"])'), - ('bar', 'bare_test(name="bar",data=["@quux"])'), - ('quux1', 'bare_test(name="quux")'), - ('quux2', 'bare_test(name="quux")'), - ]: - projects_dir.joinpath(dir_name).mkdir(exist_ok=True) - scratchFile(projects_dir.joinpath(dir_name, 'WORKSPACE')) - scratchFile( - projects_dir.joinpath(dir_name, 'BUILD'), [ - 'load("@bare_rule//:defs.bzl", "bare_test")', - 'package(default_visibility=["//visibility:public"])', - build_file, - ]) - - # We use a shell script to check that the binary itself can see the repo - # mapping manifest. This obviously doesn't work on Windows, so we just build - # the target. TODO(wyv): make this work on Windows by using Batch. - # On Linux and macOS, the script is executed in the sandbox, so we verify - # that the repository mapping is present in it. - bazel_command = 'build' if self.IsWindows() else 'test' - - # Finally we get to build stuff! - exit_code, stderr, stdout = self.RunBazel( - [bazel_command, '//:me', '--test_output=errors'], allow_failure=True) - self.AssertExitCode(0, exit_code, stderr, stdout) - - paths = ['bazel-bin/me.repo_mapping'] - if not self.IsWindows(): - paths.append('bazel-bin/me.runfiles/_repo_mapping') - for path in paths: - with open(self.Path(path), 'r') as f: - self.assertEqual( - f.read().strip(), """,foo,foo~1.0 -,me,_main -,me_ws,_main -foo~1.0,foo,foo~1.0 -foo~1.0,quux,quux~2.0 -quux~2.0,quux,quux~2.0""") - with open(self.Path('bazel-bin/me.runfiles_manifest')) as f: - self.assertIn('_repo_mapping ', f.read()) - - exit_code, stderr, stdout = self.RunBazel( - [bazel_command, '@bar//:bar', '--test_output=errors'], - allow_failure=True) - self.AssertExitCode(0, exit_code, stderr, stdout) - - paths = ['bazel-bin/external/bar~2.0/bar.repo_mapping'] - if not self.IsWindows(): - paths.append('bazel-bin/external/bar~2.0/bar.runfiles/_repo_mapping') - for path in paths: - with open(self.Path(path), 'r') as f: - self.assertEqual( - f.read().strip(), """bar~2.0,bar,bar~2.0 -bar~2.0,quux,quux~2.0 -quux~2.0,quux,quux~2.0""") - with open( - self.Path('bazel-bin/external/bar~2.0/bar.runfiles_manifest')) as f: - self.assertIn('_repo_mapping ', f.read()) - - def testJavaRunfilesLibraryRepoMapping(self): - self.main_registry.setModuleBasePath('projects') - projects_dir = self.main_registry.projects - - self.main_registry.createLocalPathModule('data', '1.0', 'data') - projects_dir.joinpath('data').mkdir(exist_ok=True) - scratchFile(projects_dir.joinpath('data', 'WORKSPACE')) - scratchFile(projects_dir.joinpath('data', 'foo.txt'), ['hello']) - scratchFile( - projects_dir.joinpath('data', 'BUILD'), ['exports_files(["foo.txt"])']) - - self.main_registry.createLocalPathModule('test', '1.0', 'test', - {'data': '1.0'}) - projects_dir.joinpath('test').mkdir(exist_ok=True) - scratchFile(projects_dir.joinpath('test', 'WORKSPACE')) - scratchFile( - projects_dir.joinpath('test', 'BUILD'), [ - 'java_test(', - ' name = "test",', - ' srcs = ["Test.java"],', - ' main_class = "com.example.Test",', - ' use_testrunner = False,', - ' data = ["@data//:foo.txt"],', - ' args = ["$(rlocationpath @data//:foo.txt)"],', - ' deps = ["@bazel_tools//tools/java/runfiles"],', - ')', - ]) - scratchFile( - projects_dir.joinpath('test', 'Test.java'), [ - 'package com.example;', - '', - 'import com.google.devtools.build.runfiles.AutoBazelRepository;', - 'import com.google.devtools.build.runfiles.Runfiles;', - '', - 'import java.io.File;', - 'import java.io.IOException;', - '', - '@AutoBazelRepository', - 'public class Test {', - ' public static void main(String[] args) throws IOException {', - ' Runfiles.Preloaded rp = Runfiles.preload();', - ' if (!new File(rp.unmapped().rlocation(args[0])).exists()) {', - ' System.exit(1);', - ' }', - ' if (!new File(rp.withSourceRepository(AutoBazelRepository_Test.NAME).rlocation("data/foo.txt")).exists()) {', - ' System.exit(1);', - ' }', - ' }', - '}', - ]) - - self.ScratchFile('MODULE.bazel', ['bazel_dep(name="test",version="1.0")']) - self.ScratchFile('WORKSPACE') - - # Run sandboxed on Linux and macOS. - exit_code, stderr, stdout = self.RunBazel([ - 'test', '@test//:test', '--test_output=errors', - '--test_env=RUNFILES_LIB_DEBUG=1' - ], - allow_failure=True) - self.AssertExitCode(exit_code, 0, stderr, stdout) - # Run unsandboxed on all platforms. - exit_code, stderr, stdout = self.RunBazel( - ['run', '@test//:test'], - allow_failure=True, - env_add={'RUNFILES_LIB_DEBUG': '1'}) - self.AssertExitCode(exit_code, 0, stderr, stdout) - - def testCppRunfilesLibraryRepoMapping(self): - self.main_registry.setModuleBasePath('projects') - projects_dir = self.main_registry.projects - - self.main_registry.createLocalPathModule('data', '1.0', 'data') - projects_dir.joinpath('data').mkdir(exist_ok=True) - scratchFile(projects_dir.joinpath('data', 'WORKSPACE')) - scratchFile(projects_dir.joinpath('data', 'foo.txt'), ['hello']) - scratchFile( - projects_dir.joinpath('data', 'BUILD'), ['exports_files(["foo.txt"])']) - - self.main_registry.createLocalPathModule('test', '1.0', 'test', - {'data': '1.0'}) - projects_dir.joinpath('test').mkdir(exist_ok=True) - scratchFile(projects_dir.joinpath('test', 'WORKSPACE')) - scratchFile( - projects_dir.joinpath('test', 'BUILD'), [ - 'cc_test(', - ' name = "test",', - ' srcs = ["test.cpp"],', - ' data = ["@data//:foo.txt"],', - ' args = ["$(rlocationpath @data//:foo.txt)"],', - ' deps = ["@bazel_tools//tools/cpp/runfiles"],', - ')', - ]) - scratchFile( - projects_dir.joinpath('test', 'test.cpp'), [ - '#include ', - '#include ', - '#include "tools/cpp/runfiles/runfiles.h"', - 'using bazel::tools::cpp::runfiles::Runfiles;', - 'int main(int argc, char** argv) {', - ' Runfiles* runfiles = Runfiles::Create(argv[0], BAZEL_CURRENT_REPOSITORY);', - ' std::ifstream f1(runfiles->Rlocation(argv[1]));', - ' if (!f1.good()) std::exit(1);', - ' std::ifstream f2(runfiles->Rlocation("data/foo.txt"));', - ' if (!f2.good()) std::exit(2);', - '}', - ]) - - self.ScratchFile('MODULE.bazel', ['bazel_dep(name="test",version="1.0")']) - self.ScratchFile('WORKSPACE') - - # Run sandboxed on Linux and macOS. - exit_code, stderr, stdout = self.RunBazel( - ['test', '@test//:test', '--test_output=errors'], allow_failure=True) - self.AssertExitCode(exit_code, 0, stderr, stdout) - # Run unsandboxed on all platforms. - exit_code, stderr, stdout = self.RunBazel(['run', '@test//:test'], - allow_failure=True) - self.AssertExitCode(exit_code, 0, stderr, stdout) - - def testBashRunfilesLibraryRepoMapping(self): - self.main_registry.setModuleBasePath('projects') - projects_dir = self.main_registry.projects - - self.main_registry.createLocalPathModule('data', '1.0', 'data') - projects_dir.joinpath('data').mkdir(exist_ok=True) - scratchFile(projects_dir.joinpath('data', 'WORKSPACE')) - scratchFile(projects_dir.joinpath('data', 'foo.txt'), ['hello']) - scratchFile( - projects_dir.joinpath('data', 'BUILD'), ['exports_files(["foo.txt"])']) - - self.main_registry.createLocalPathModule('test', '1.0', 'test', - {'data': '1.0'}) - projects_dir.joinpath('test').mkdir(exist_ok=True) - scratchFile(projects_dir.joinpath('test', 'WORKSPACE')) - scratchFile( - projects_dir.joinpath('test', 'BUILD'), [ - 'sh_test(', - ' name = "test",', - ' srcs = ["test.sh"],', - ' data = ["@data//:foo.txt"],', - ' args = ["$(rlocationpath @data//:foo.txt)"],', - ' deps = ["@bazel_tools//tools/bash/runfiles"],', - ')', - ]) - test_script = projects_dir.joinpath('test', 'test.sh') - scratchFile( - test_script, """#!/usr/bin/env bash -# --- begin runfiles.bash initialization v2 --- -# Copy-pasted from the Bazel Bash runfiles library v2. -set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash -source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ - source "$0.runfiles/$f" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ - source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ - { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e -# --- end runfiles.bash initialization v2 --- -[[ -f "$(rlocation $1)" ]] || exit 1 -[[ -f "$(rlocation data/foo.txt)" ]] || exit 2 -""".splitlines()) - os.chmod(test_script, 0o755) - - self.ScratchFile('MODULE.bazel', ['bazel_dep(name="test",version="1.0")']) - self.ScratchFile('WORKSPACE') - - # Run sandboxed on Linux and macOS. - exit_code, stderr, stdout = self.RunBazel([ - 'test', '@test//:test', '--test_output=errors', - '--test_env=RUNFILES_LIB_DEBUG=1' - ], - allow_failure=True) - self.AssertExitCode(exit_code, 0, stderr, stdout) - # Run unsandboxed on all platforms. - exit_code, stderr, stdout = self.RunBazel( - ['run', '@test//:test'], - allow_failure=True, - env_add={'RUNFILES_LIB_DEBUG': '1'}) - self.AssertExitCode(exit_code, 0, stderr, stdout) - def testNativePackageRelativeLabel(self): self.ScratchFile( 'MODULE.bazel', diff --git a/src/test/py/bazel/bzlmod/bazel_overrides_test.py b/src/test/py/bazel/bzlmod/bazel_overrides_test.py new file mode 100644 index 00000000000000..9ec48c374e579a --- /dev/null +++ b/src/test/py/bazel/bzlmod/bazel_overrides_test.py @@ -0,0 +1,364 @@ +# pylint: disable=g-backslash-continuation +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=g-long-ternary + +import os +import tempfile +import unittest + +from src.test.py.bazel import test_base +from src.test.py.bazel.bzlmod.test_utils import BazelRegistry + + +class BazelOverridesTest(test_base.TestBase): + + def setUp(self): + test_base.TestBase.setUp(self) + self.registries_work_dir = tempfile.mkdtemp(dir=self._test_cwd) + self.main_registry = BazelRegistry( + os.path.join(self.registries_work_dir, 'main') + ) + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'aaa', '1.1' + ).createCcModule('bbb', '1.0', {'aaa': '1.0'}).createCcModule( + 'bbb', '1.1', {'aaa': '1.1'} + ).createCcModule( + 'ccc', '1.1', {'aaa': '1.1', 'bbb': '1.1'} + ) + self.ScratchFile( + '.bazelrc', + [ + # In ipv6 only network, this has to be enabled. + # 'startup --host_jvm_args=-Djava.net.preferIPv6Addresses=true', + 'common --enable_bzlmod', + 'common --registry=' + self.main_registry.getURL(), + # We need to have BCR here to make sure built-in modules like + # bazel_tools can work. + 'common --registry=https://bcr.bazel.build', + 'common --verbose_failures', + # Set an explicit Java language version + 'common --java_language_version=8', + 'common --tool_java_language_version=8', + 'common --lockfile_mode=update', + ], + ) + self.ScratchFile('WORKSPACE') + # The existence of WORKSPACE.bzlmod prevents WORKSPACE prefixes or suffixes + # from being used; this allows us to test built-in modules actually work + self.ScratchFile('WORKSPACE.bzlmod') + + def writeMainProjectFiles(self): + self.ScratchFile( + 'aaa.patch', + [ + '--- a/aaa.cc', + '+++ b/aaa.cc', + '@@ -1,6 +1,6 @@', + ' #include ', + ' #include "aaa.h"', + ' void hello_aaa(const std::string& caller) {', + '- std::string lib_name = "aaa@1.0";', + '+ std::string lib_name = "aaa@1.0 (locally patched)";', + ' printf("%s => %s\\n", caller.c_str(), lib_name.c_str());', + ' }', + ], + ) + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = [', + ' "@aaa//:lib_aaa",', + ' "@bbb//:lib_bbb",', + ' ],', + ')', + ], + ) + self.ScratchFile( + 'main.cc', + [ + '#include "aaa.h"', + '#include "bbb.h"', + 'int main() {', + ' hello_aaa("main function");', + ' hello_bbb("main function");', + '}', + ], + ) + + def testSingleVersionOverrideWithPatch(self): + self.writeMainProjectFiles() + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "aaa", version = "1.1")', + 'bazel_dep(name = "bbb", version = "1.1")', + # Both main and bbb@1.1 has to depend on the locally patched aaa@1.0 + 'single_version_override(', + ' module_name = "aaa",', + ' version = "1.0",', + ' patches = ["//:aaa.patch"],', + ' patch_strip = 1,', + ')', + ], + ) + _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) + self.assertIn('main function => aaa@1.0 (locally patched)', stdout) + self.assertIn('main function => bbb@1.1', stdout) + self.assertIn('bbb@1.1 => aaa@1.0 (locally patched)', stdout) + + def testRegistryOverride(self): + self.writeMainProjectFiles() + another_registry = BazelRegistry( + os.path.join(self.registries_work_dir, 'another'), + ' from another registry', + ) + another_registry.createCcModule('aaa', '1.0') + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "aaa", version = "1.0")', + 'bazel_dep(name = "bbb", version = "1.0")', + 'single_version_override(', + ' module_name = "aaa",', + ' registry = "%s",' % another_registry.getURL(), + ')', + ], + ) + _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) + self.assertIn('main function => aaa@1.0 from another registry', stdout) + self.assertIn('main function => bbb@1.0', stdout) + self.assertIn('bbb@1.0 => aaa@1.0 from another registry', stdout) + + def testArchiveOverride(self): + self.writeMainProjectFiles() + archive_aaa_1_0 = self.main_registry.archives.joinpath('aaa.1.0.zip') + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "aaa", version = "1.1")', + 'bazel_dep(name = "bbb", version = "1.1")', + 'archive_override(', + ' module_name = "aaa",', + ' urls = ["%s"],' % archive_aaa_1_0.as_uri(), + ' patches = ["//:aaa.patch"],', + ' patch_strip = 1,', + ')', + ], + ) + _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) + self.assertIn('main function => aaa@1.0 (locally patched)', stdout) + self.assertIn('main function => bbb@1.1', stdout) + self.assertIn('bbb@1.1 => aaa@1.0 (locally patched)', stdout) + + def testGitOverride(self): + self.writeMainProjectFiles() + src_aaa_1_0 = self.main_registry.projects.joinpath('aaa', '1.0') + self.RunProgram(['git', 'init'], cwd=src_aaa_1_0, allow_failure=False) + self.RunProgram( + ['git', 'config', 'user.name', 'tester'], + cwd=src_aaa_1_0, + allow_failure=False, + ) + self.RunProgram( + ['git', 'config', 'user.email', 'tester@foo.com'], + cwd=src_aaa_1_0, + allow_failure=False, + ) + self.RunProgram(['git', 'add', './'], cwd=src_aaa_1_0, allow_failure=False) + self.RunProgram( + ['git', 'commit', '-m', 'Initial commit.'], + cwd=src_aaa_1_0, + allow_failure=False, + ) + _, stdout, _ = self.RunProgram( + ['git', 'rev-parse', 'HEAD'], cwd=src_aaa_1_0, allow_failure=False + ) + commit = stdout[0].strip() + + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "aaa", version = "1.1")', + 'bazel_dep(name = "bbb", version = "1.1")', + 'git_override(', + ' module_name = "aaa",', + ' remote = "%s",' % src_aaa_1_0.as_uri(), + ' commit = "%s",' % commit, + ' patches = ["//:aaa.patch"],', + ' patch_strip = 1,', + ')', + ], + ) + _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) + self.assertIn('main function => aaa@1.0 (locally patched)', stdout) + self.assertIn('main function => bbb@1.1', stdout) + self.assertIn('bbb@1.1 => aaa@1.0 (locally patched)', stdout) + + def testLocalPathOverride(self): + src_aaa_1_0 = self.main_registry.projects.joinpath('aaa', '1.0') + self.writeMainProjectFiles() + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "aaa", version = "1.1")', + 'bazel_dep(name = "bbb", version = "1.1")', + 'local_path_override(', + ' module_name = "aaa",', + ' path = "%s",' % str(src_aaa_1_0.resolve()).replace('\\', '/'), + ')', + ], + ) + _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False) + self.assertIn('main function => aaa@1.0', stdout) + self.assertIn('main function => bbb@1.1', stdout) + self.assertIn('bbb@1.1 => aaa@1.0', stdout) + + def testCmdAbsoluteModuleOverride(self): + # test commandline_overrides takes precedence over local_path_override + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "ss", version = "1.0")', + 'local_path_override(', + ' module_name = "ss",', + ' path = "%s",' % self.Path('aa'), + ')', + ], + ) + self.ScratchFile('BUILD') + self.ScratchFile('WORKSPACE') + + self.ScratchFile( + 'aa/MODULE.bazel', + [ + "module(name='ss')", + ], + ) + self.ScratchFile( + 'aa/BUILD', + [ + 'filegroup(name = "never_ever")', + ], + ) + self.ScratchFile('aa/WORKSPACE') + + self.ScratchFile( + 'bb/MODULE.bazel', + [ + "module(name='ss')", + ], + ) + self.ScratchFile( + 'bb/BUILD', + [ + 'filegroup(name = "choose_me")', + ], + ) + self.ScratchFile('bb/WORKSPACE') + + _, _, stderr = self.RunBazel( + ['build', '@ss//:all', '--override_module', 'ss=' + self.Path('bb')], + allow_failure=False, + ) + # module file override should be ignored, and bb directory should be used + self.assertIn( + 'Target @ss~override//:choose_me up-to-date (nothing to build)', stderr + ) + + def testCmdRelativeModuleOverride(self): + self.ScratchFile('aa/WORKSPACE') + self.ScratchFile( + 'aa/MODULE.bazel', + [ + 'bazel_dep(name = "ss", version = "1.0")', + ], + ) + self.ScratchFile('aa/BUILD') + + self.ScratchFile('aa/cc/BUILD') + + self.ScratchFile('bb/WORKSPACE') + self.ScratchFile( + 'bb/MODULE.bazel', + [ + 'module(name="ss")', + ], + ) + self.ScratchFile( + 'bb/BUILD', + [ + 'filegroup(name = "choose_me")', + ], + ) + + _, _, stderr = self.RunBazel( + [ + 'build', + '@ss//:all', + '--override_module', + 'ss=../../bb', + '--enable_bzlmod', + ], + cwd=self.Path('aa/cc'), + allow_failure=False, + ) + self.assertIn( + 'Target @ss~override//:choose_me up-to-date (nothing to build)', stderr + ) + + def testCmdWorkspaceRelativeModuleOverride(self): + self.ScratchFile('WORKSPACE') + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "ss", version = "1.0")', + ], + ) + self.ScratchFile('BUILD') + self.ScratchFile('aa/BUILD') + self.ScratchFile('bb/WORKSPACE') + self.ScratchFile( + 'bb/MODULE.bazel', + [ + 'module(name="ss")', + ], + ) + self.ScratchFile( + 'bb/BUILD', + [ + 'filegroup(name = "choose_me")', + ], + ) + + _, _, stderr = self.RunBazel( + [ + 'build', + '@ss//:all', + '--override_module', + 'ss=%workspace%/bb', + ], + cwd=self.Path('aa'), + allow_failure=False, + ) + self.assertIn( + 'Target @ss~override//:choose_me up-to-date (nothing to build)', stderr + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/test/py/bazel/bzlmod/bazel_repo_mapping_test.py b/src/test/py/bazel/bzlmod/bazel_repo_mapping_test.py new file mode 100644 index 00000000000000..2d992d044fa9c6 --- /dev/null +++ b/src/test/py/bazel/bzlmod/bazel_repo_mapping_test.py @@ -0,0 +1,420 @@ +# pylint: disable=g-backslash-continuation +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=g-long-ternary + +import os +import tempfile +import unittest + +from src.test.py.bazel import test_base +from src.test.py.bazel.bzlmod.test_utils import BazelRegistry +from src.test.py.bazel.bzlmod.test_utils import scratchFile + + +class BazelRepoMappingTest(test_base.TestBase): + + def setUp(self): + test_base.TestBase.setUp(self) + self.registries_work_dir = tempfile.mkdtemp(dir=self._test_cwd) + self.main_registry = BazelRegistry( + os.path.join(self.registries_work_dir, 'main') + ) + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'aaa', '1.1' + ).createCcModule('bbb', '1.0', {'aaa': '1.0'}).createCcModule( + 'bbb', '1.1', {'aaa': '1.1'} + ).createCcModule( + 'ccc', '1.1', {'aaa': '1.1', 'bbb': '1.1'} + ) + self.ScratchFile( + '.bazelrc', + [ + # In ipv6 only network, this has to be enabled. + # 'startup --host_jvm_args=-Djava.net.preferIPv6Addresses=true', + 'common --enable_bzlmod', + 'common --registry=' + self.main_registry.getURL(), + # We need to have BCR here to make sure built-in modules like + # bazel_tools can work. + 'common --registry=https://bcr.bazel.build', + 'common --verbose_failures', + # Set an explicit Java language version + 'common --java_language_version=8', + 'common --tool_java_language_version=8', + 'common --lockfile_mode=update', + ], + ) + self.ScratchFile('WORKSPACE') + # The existence of WORKSPACE.bzlmod prevents WORKSPACE prefixes or suffixes + # from being used; this allows us to test built-in modules actually work + self.ScratchFile('WORKSPACE.bzlmod') + + def testRunfilesRepoMappingManifest(self): + self.main_registry.setModuleBasePath('projects') + projects_dir = self.main_registry.projects + + # Set up a "bare_rule" module that contains the "bare_test" rule which + # passes runfiles along + self.main_registry.createLocalPathModule('bare_rule', '1.0', 'bare_rule') + projects_dir.joinpath('bare_rule').mkdir(exist_ok=True) + scratchFile(projects_dir.joinpath('bare_rule', 'WORKSPACE')) + scratchFile(projects_dir.joinpath('bare_rule', 'BUILD')) + # The working directory of a test is the subdirectory of the runfiles + # directory corresponding to the main repository. + scratchFile( + projects_dir.joinpath('bare_rule', 'defs.bzl'), + [ + 'def _bare_test_impl(ctx):', + ' exe = ctx.actions.declare_file(ctx.label.name)', + ' ctx.actions.write(exe,', + ( + ' "#/bin/bash\\nif [[ ! -f ../_repo_mapping || ! -s' + ' ../_repo_mapping ]]; then\\necho >&2 \\"ERROR: cannot find' + ' repo mapping manifest file\\"\\nexit 1\\nfi",' + ), + ' True)', + ' runfiles = ctx.runfiles(files=ctx.files.data)', + ' for data in ctx.attr.data:', + ' runfiles = runfiles.merge(data[DefaultInfo].default_runfiles)', + ( + ' return DefaultInfo(files=depset(direct=[exe]),' + ' executable=exe, runfiles=runfiles)' + ), + 'bare_test=rule(', + ' implementation=_bare_test_impl,', + ' attrs={"data":attr.label_list(allow_files=True)},', + ' test=True,', + ')', + ], + ) + + # Now set up a project tree shaped like a diamond + self.ScratchFile( + 'MODULE.bazel', + [ + 'module(name="me",version="1.0")', + 'bazel_dep(name="foo",version="1.0")', + 'bazel_dep(name="bar",version="2.0")', + 'bazel_dep(name="bare_rule",version="1.0")', + ], + ) + self.ScratchFile('WORKSPACE') + self.ScratchFile('WORKSPACE.bzlmod', ['workspace(name="me_ws")']) + self.ScratchFile( + 'BUILD', + [ + 'load("@bare_rule//:defs.bzl", "bare_test")', + 'bare_test(name="me",data=["@foo"])', + ], + ) + self.main_registry.createLocalPathModule( + 'foo', '1.0', 'foo', {'quux': '1.0', 'bare_rule': '1.0'} + ) + self.main_registry.createLocalPathModule( + 'bar', '2.0', 'bar', {'quux': '2.0', 'bare_rule': '1.0'} + ) + self.main_registry.createLocalPathModule( + 'quux', '1.0', 'quux1', {'bare_rule': '1.0'} + ) + self.main_registry.createLocalPathModule( + 'quux', '2.0', 'quux2', {'bare_rule': '1.0'} + ) + for dir_name, build_file in [ + ('foo', 'bare_test(name="foo",data=["@quux"])'), + ('bar', 'bare_test(name="bar",data=["@quux"])'), + ('quux1', 'bare_test(name="quux")'), + ('quux2', 'bare_test(name="quux")'), + ]: + projects_dir.joinpath(dir_name).mkdir(exist_ok=True) + scratchFile(projects_dir.joinpath(dir_name, 'WORKSPACE')) + scratchFile( + projects_dir.joinpath(dir_name, 'BUILD'), + [ + 'load("@bare_rule//:defs.bzl", "bare_test")', + 'package(default_visibility=["//visibility:public"])', + build_file, + ], + ) + + # We use a shell script to check that the binary itself can see the repo + # mapping manifest. This obviously doesn't work on Windows, so we just build + # the target. TODO(wyv): make this work on Windows by using Batch. + # On Linux and macOS, the script is executed in the sandbox, so we verify + # that the repository mapping is present in it. + bazel_command = 'build' if self.IsWindows() else 'test' + + # Finally we get to build stuff! + exit_code, stderr, stdout = self.RunBazel( + [bazel_command, '//:me', '--test_output=errors'], allow_failure=True + ) + self.AssertExitCode(0, exit_code, stderr, stdout) + + paths = ['bazel-bin/me.repo_mapping'] + if not self.IsWindows(): + paths.append('bazel-bin/me.runfiles/_repo_mapping') + for path in paths: + with open(self.Path(path), 'r') as f: + self.assertEqual( + f.read().strip(), + """,foo,foo~1.0 +,me,_main +,me_ws,_main +foo~1.0,foo,foo~1.0 +foo~1.0,quux,quux~2.0 +quux~2.0,quux,quux~2.0""", + ) + with open(self.Path('bazel-bin/me.runfiles_manifest')) as f: + self.assertIn('_repo_mapping ', f.read()) + + exit_code, stderr, stdout = self.RunBazel( + [bazel_command, '@bar//:bar', '--test_output=errors'], + allow_failure=True, + ) + self.AssertExitCode(0, exit_code, stderr, stdout) + + paths = ['bazel-bin/external/bar~2.0/bar.repo_mapping'] + if not self.IsWindows(): + paths.append('bazel-bin/external/bar~2.0/bar.runfiles/_repo_mapping') + for path in paths: + with open(self.Path(path), 'r') as f: + self.assertEqual( + f.read().strip(), + """bar~2.0,bar,bar~2.0 +bar~2.0,quux,quux~2.0 +quux~2.0,quux,quux~2.0""", + ) + with open( + self.Path('bazel-bin/external/bar~2.0/bar.runfiles_manifest') + ) as f: + self.assertIn('_repo_mapping ', f.read()) + + def testBashRunfilesLibraryRepoMapping(self): + self.main_registry.setModuleBasePath('projects') + projects_dir = self.main_registry.projects + + self.main_registry.createLocalPathModule('data', '1.0', 'data') + projects_dir.joinpath('data').mkdir(exist_ok=True) + scratchFile(projects_dir.joinpath('data', 'WORKSPACE')) + scratchFile(projects_dir.joinpath('data', 'foo.txt'), ['hello']) + scratchFile( + projects_dir.joinpath('data', 'BUILD'), ['exports_files(["foo.txt"])'] + ) + + self.main_registry.createLocalPathModule( + 'test', '1.0', 'test', {'data': '1.0'} + ) + projects_dir.joinpath('test').mkdir(exist_ok=True) + scratchFile(projects_dir.joinpath('test', 'WORKSPACE')) + scratchFile( + projects_dir.joinpath('test', 'BUILD'), + [ + 'sh_test(', + ' name = "test",', + ' srcs = ["test.sh"],', + ' data = ["@data//:foo.txt"],', + ' args = ["$(rlocationpath @data//:foo.txt)"],', + ' deps = ["@bazel_tools//tools/bash/runfiles"],', + ')', + ], + ) + test_script = projects_dir.joinpath('test', 'test.sh') + scratchFile( + test_script, + """#!/usr/bin/env bash +# --- begin runfiles.bash initialization v2 --- +# Copy-pasted from the Bazel Bash runfiles library v2. +set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v2 --- +[[ -f "$(rlocation $1)" ]] || exit 1 +[[ -f "$(rlocation data/foo.txt)" ]] || exit 2 +""".splitlines(), + ) + os.chmod(test_script, 0o755) + + self.ScratchFile('MODULE.bazel', ['bazel_dep(name="test",version="1.0")']) + self.ScratchFile('WORKSPACE') + + # Run sandboxed on Linux and macOS. + exit_code, stderr, stdout = self.RunBazel( + [ + 'test', + '@test//:test', + '--test_output=errors', + '--test_env=RUNFILES_LIB_DEBUG=1', + ], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 0, stderr, stdout) + # Run unsandboxed on all platforms. + exit_code, stderr, stdout = self.RunBazel( + ['run', '@test//:test'], + allow_failure=True, + env_add={'RUNFILES_LIB_DEBUG': '1'}, + ) + self.AssertExitCode(exit_code, 0, stderr, stdout) + + def testCppRunfilesLibraryRepoMapping(self): + self.main_registry.setModuleBasePath('projects') + projects_dir = self.main_registry.projects + + self.main_registry.createLocalPathModule('data', '1.0', 'data') + projects_dir.joinpath('data').mkdir(exist_ok=True) + scratchFile(projects_dir.joinpath('data', 'WORKSPACE')) + scratchFile(projects_dir.joinpath('data', 'foo.txt'), ['hello']) + scratchFile( + projects_dir.joinpath('data', 'BUILD'), ['exports_files(["foo.txt"])'] + ) + + self.main_registry.createLocalPathModule( + 'test', '1.0', 'test', {'data': '1.0'} + ) + projects_dir.joinpath('test').mkdir(exist_ok=True) + scratchFile(projects_dir.joinpath('test', 'WORKSPACE')) + scratchFile( + projects_dir.joinpath('test', 'BUILD'), + [ + 'cc_test(', + ' name = "test",', + ' srcs = ["test.cpp"],', + ' data = ["@data//:foo.txt"],', + ' args = ["$(rlocationpath @data//:foo.txt)"],', + ' deps = ["@bazel_tools//tools/cpp/runfiles"],', + ')', + ], + ) + scratchFile( + projects_dir.joinpath('test', 'test.cpp'), + [ + '#include ', + '#include ', + '#include "tools/cpp/runfiles/runfiles.h"', + 'using bazel::tools::cpp::runfiles::Runfiles;', + 'int main(int argc, char** argv) {', + ( + ' Runfiles* runfiles = Runfiles::Create(argv[0],' + ' BAZEL_CURRENT_REPOSITORY);' + ), + ' std::ifstream f1(runfiles->Rlocation(argv[1]));', + ' if (!f1.good()) std::exit(1);', + ' std::ifstream f2(runfiles->Rlocation("data/foo.txt"));', + ' if (!f2.good()) std::exit(2);', + '}', + ], + ) + + self.ScratchFile('MODULE.bazel', ['bazel_dep(name="test",version="1.0")']) + self.ScratchFile('WORKSPACE') + + # Run sandboxed on Linux and macOS. + exit_code, stderr, stdout = self.RunBazel( + ['test', '@test//:test', '--test_output=errors'], allow_failure=True + ) + self.AssertExitCode(exit_code, 0, stderr, stdout) + # Run unsandboxed on all platforms. + exit_code, stderr, stdout = self.RunBazel( + ['run', '@test//:test'], allow_failure=True + ) + self.AssertExitCode(exit_code, 0, stderr, stdout) + + def testJavaRunfilesLibraryRepoMapping(self): + self.main_registry.setModuleBasePath('projects') + projects_dir = self.main_registry.projects + + self.main_registry.createLocalPathModule('data', '1.0', 'data') + projects_dir.joinpath('data').mkdir(exist_ok=True) + scratchFile(projects_dir.joinpath('data', 'WORKSPACE')) + scratchFile(projects_dir.joinpath('data', 'foo.txt'), ['hello']) + scratchFile( + projects_dir.joinpath('data', 'BUILD'), ['exports_files(["foo.txt"])'] + ) + + self.main_registry.createLocalPathModule( + 'test', '1.0', 'test', {'data': '1.0'} + ) + projects_dir.joinpath('test').mkdir(exist_ok=True) + scratchFile(projects_dir.joinpath('test', 'WORKSPACE')) + scratchFile( + projects_dir.joinpath('test', 'BUILD'), + [ + 'java_test(', + ' name = "test",', + ' srcs = ["Test.java"],', + ' main_class = "com.example.Test",', + ' use_testrunner = False,', + ' data = ["@data//:foo.txt"],', + ' args = ["$(rlocationpath @data//:foo.txt)"],', + ' deps = ["@bazel_tools//tools/java/runfiles"],', + ')', + ], + ) + scratchFile( + projects_dir.joinpath('test', 'Test.java'), + [ + 'package com.example;', + '', + 'import com.google.devtools.build.runfiles.AutoBazelRepository;', + 'import com.google.devtools.build.runfiles.Runfiles;', + '', + 'import java.io.File;', + 'import java.io.IOException;', + '', + '@AutoBazelRepository', + 'public class Test {', + ' public static void main(String[] args) throws IOException {', + ' Runfiles.Preloaded rp = Runfiles.preload();', + ' if (!new File(rp.unmapped().rlocation(args[0])).exists()) {', + ' System.exit(1);', + ' }', + ( + ' if (!new' + ' File(rp.withSourceRepository(AutoBazelRepository_Test.NAME).rlocation("data/foo.txt")).exists()) {' + ), + ' System.exit(1);', + ' }', + ' }', + '}', + ], + ) + + self.ScratchFile('MODULE.bazel', ['bazel_dep(name="test",version="1.0")']) + self.ScratchFile('WORKSPACE') + + # Run sandboxed on Linux and macOS. + exit_code, stderr, stdout = self.RunBazel( + [ + 'test', + '@test//:test', + '--test_output=errors', + '--test_env=RUNFILES_LIB_DEBUG=1', + ], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 0, stderr, stdout) + # Run unsandboxed on all platforms. + exit_code, stderr, stdout = self.RunBazel( + ['run', '@test//:test'], + allow_failure=True, + env_add={'RUNFILES_LIB_DEBUG': '1'}, + ) + self.AssertExitCode(exit_code, 0, stderr, stdout) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/test/py/bazel/bzlmod/bazel_yanked_versions_test.py b/src/test/py/bazel/bzlmod/bazel_yanked_versions_test.py new file mode 100644 index 00000000000000..52fa8970503d0f --- /dev/null +++ b/src/test/py/bazel/bzlmod/bazel_yanked_versions_test.py @@ -0,0 +1,240 @@ +# pylint: disable=g-backslash-continuation +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# pylint: disable=g-long-ternary + +import os +import tempfile +import unittest + +from src.test.py.bazel import test_base +from src.test.py.bazel.bzlmod.test_utils import BazelRegistry + + +class BazelYankedVersionsTest(test_base.TestBase): + + def setUp(self): + test_base.TestBase.setUp(self) + self.registries_work_dir = tempfile.mkdtemp(dir=self._test_cwd) + self.main_registry = BazelRegistry( + os.path.join(self.registries_work_dir, 'main') + ) + self.main_registry.createCcModule('aaa', '1.0').createCcModule( + 'aaa', '1.1' + ).createCcModule('bbb', '1.0', {'aaa': '1.0'}).createCcModule( + 'bbb', '1.1', {'aaa': '1.1'} + ).createCcModule( + 'ccc', '1.1', {'aaa': '1.1', 'bbb': '1.1'} + ).createCcModule( + 'ddd', '1.0', {'yanked1': '1.0', 'yanked2': '1.0'} + ).createCcModule( + 'eee', '1.0', {'yanked1': '1.0'} + ).createCcModule( + 'yanked1', '1.0' + ).createCcModule( + 'yanked2', '1.0' + ).addMetadata( + 'yanked1', yanked_versions={'1.0': 'dodgy'} + ).addMetadata( + 'yanked2', yanked_versions={'1.0': 'sketchy'} + ) + self.writeBazelrcFile() + self.ScratchFile('WORKSPACE') + # The existence of WORKSPACE.bzlmod prevents WORKSPACE prefixes or suffixes + # from being used; this allows us to test built-in modules actually work + self.ScratchFile('WORKSPACE.bzlmod') + + def writeBazelrcFile(self, allow_yanked_versions=True): + self.ScratchFile( + '.bazelrc', + [ + # In ipv6 only network, this has to be enabled. + # 'startup --host_jvm_args=-Djava.net.preferIPv6Addresses=true', + 'common --enable_bzlmod', + 'common --registry=' + self.main_registry.getURL(), + # We need to have BCR here to make sure built-in modules like + # bazel_tools can work. + 'common --registry=https://bcr.bazel.build', + 'common --verbose_failures', + # Set an explicit Java language version + 'common --java_language_version=8', + 'common --tool_java_language_version=8', + 'common --lockfile_mode=update', + ] + + ( + [ + # Disable yanked version check so we are not affected BCR + # changes. + 'common --allow_yanked_versions=all', + ] + if allow_yanked_versions + else [] + ), + ) + + def testNonRegistryOverriddenModulesIgnoreYanked(self): + self.writeBazelrcFile(allow_yanked_versions=False) + src_yanked1 = self.main_registry.projects.joinpath('yanked1', '1.0') + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "yanked1", version = "1.0")', + 'local_path_override(', + ' module_name = "yanked1",', + ' path = "%s",' % str(src_yanked1.resolve()).replace('\\', '/'), + ')', + ], + ) + self.ScratchFile('WORKSPACE') + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@yanked1//:lib_yanked1"],', + ')', + ], + ) + self.RunBazel(['build', '--nobuild', '//:main'], allow_failure=False) + + def testContainingYankedDepFails(self): + self.writeBazelrcFile(allow_yanked_versions=False) + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "yanked1", version = "1.0")', + ], + ) + self.ScratchFile('WORKSPACE') + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@ddd//:lib_ddd"],', + ')', + ], + ) + exit_code, _, stderr = self.RunBazel( + ['build', '--nobuild', '//:main'], allow_failure=True + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + 'Yanked version detected in your resolved dependency graph: ' + + 'yanked1@1.0, for the reason: dodgy.', + ''.join(stderr), + ) + + def testAllowedYankedDepsSuccessByFlag(self): + self.writeBazelrcFile(allow_yanked_versions=False) + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "ddd", version = "1.0")', + ], + ) + self.ScratchFile('WORKSPACE') + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@ddd//:lib_ddd"],', + ')', + ], + ) + self.RunBazel( + [ + 'build', + '--nobuild', + '--allow_yanked_versions=yanked1@1.0,yanked2@1.0', + '//:main', + ], + allow_failure=False, + ) + + def testAllowedYankedDepsByEnvVar(self): + self.writeBazelrcFile(allow_yanked_versions=False) + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "ddd", version = "1.0")', + ], + ) + self.ScratchFile('WORKSPACE') + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@ddd//:lib_ddd"],', + ')', + ], + ) + self.RunBazel( + ['build', '--nobuild', '//:main'], + env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked1@1.0,yanked2@1.0'}, + allow_failure=False, + ) + + # Test changing the env var, the build should fail again. + exit_code, _, stderr = self.RunBazel( + ['build', '--nobuild', '//:main'], + env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked2@1.0'}, + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + 'Yanked version detected in your resolved dependency graph: ' + + 'yanked1@1.0, for the reason: dodgy.', + ''.join(stderr), + ) + + def testAllowedYankedDepsSuccessMix(self): + self.writeBazelrcFile(allow_yanked_versions=False) + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "ddd", version = "1.0")', + ], + ) + self.ScratchFile('WORKSPACE') + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@ddd//:lib_ddd"],', + ')', + ], + ) + self.RunBazel( + [ + 'build', + '--nobuild', + '--allow_yanked_versions=yanked1@1.0', + '//:main', + ], + env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked2@1.0'}, + allow_failure=False, + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/test/shell/integration/minimal_jdk_test.sh b/src/test/shell/integration/minimal_jdk_test.sh index 03f6474d107afa..b7c9a1c17c61f3 100755 --- a/src/test/shell/integration/minimal_jdk_test.sh +++ b/src/test/shell/integration/minimal_jdk_test.sh @@ -42,13 +42,13 @@ export BAZEL_SUFFIX="_jdk_minimal" source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \ || { echo "integration_test_setup.sh not found!" >&2; exit 1; } -# Bazel's install base is < 320MB with minimal JDK and > 320MB with an all +# Bazel's install base is < 342MB with minimal JDK and > 342MB with an all # modules JDK. -function test_size_less_than_320MB() { +function test_size_less_than_342MB() { bazel info ib=$(bazel info install_base) size=$(du -s "$ib" | cut -d\ -f1) - maxsize=$((1024*320)) + maxsize=$((1024*342)) if [ $size -gt $maxsize ]; then echo "$ib was too big:" 1>&2 du -a "$ib" 1>&2 diff --git a/third_party/BUILD b/third_party/BUILD index 657ce837367c26..88a6a73a4df645 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -239,9 +239,9 @@ java_plugin( distrib_java_import( name = "auto_common", + applicable_licenses = [":auto_license"], enable_distributions = ["debian"], jars = ["auto/auto-common-1.1.2.jar"], - applicable_licenses = [":auto_license"], ) license( @@ -297,13 +297,37 @@ java_plugin( ], ) +java_plugin( + name = "auto_value_gson_plugin", + processor_class = "com.ryanharter.auto.value.gson.factory.AutoValueGsonAdapterFactoryProcessor", + deps = [ + "auto/autotransient-1.0.0.jar", + ":auto_value_gson", + ":auto_value_value", + ":gson", + "//third_party/java/javapoet", + ], +) + +java_import( + name = "auto_value_gson", + applicable_licenses = [":auto_license"], + jars = [ + "auto/auto-value-gson-extension-1.3.1.jar", + "auto/auto-value-gson-factory-1.3.1.jar", + "auto/auto-value-gson-runtime-1.3.1.jar", + ], +) + java_library( name = "auto_value", exported_plugins = [ ":auto_annotation_plugin", ":auto_value_plugin", + ":auto_value_gson_plugin", ], exports = [ + ":auto_value_gson", ":auto_value_value", ":tomcat_annotations_api", ], @@ -451,6 +475,7 @@ filegroup( distrib_java_import( name = "guava", + applicable_licenses = [":guava_license"], enable_distributions = ["debian"], jars = [ "guava/failureaccess-1.0.1.jar", @@ -461,7 +486,6 @@ distrib_java_import( ":jcip_annotations", ":jsr305", ], - applicable_licenses = [":guava_license"], ) license( @@ -475,13 +499,13 @@ license( distrib_java_import( name = "flogger", + applicable_licenses = [":flogger_license"], enable_distributions = ["debian"], jars = [ "flogger/flogger-0.5.1.jar", "flogger/flogger-system-backend-0.5.1.jar", "flogger/google-extensions-0.5.1.jar", ], - applicable_licenses = [":flogger_license"], ) license( @@ -743,6 +767,7 @@ java_import( java_import( name = "truth", + applicable_licenses = [":truth_license"], jars = ["truth/truth-1.0.1.jar"], exports = [ ":truth8", @@ -751,7 +776,6 @@ java_import( deps = [ ":diffutils", ], - applicable_licenses = [":truth_license"], ) license( @@ -765,8 +789,8 @@ license( java_import( name = "truth8", - jars = ["truth8/truth-java8-extension-1.0.1.jar"], applicable_licenses = [":truth8_license"], + jars = ["truth8/truth-java8-extension-1.0.1.jar"], ) license( @@ -780,11 +804,11 @@ license( java_import( name = "truth_proto", + applicable_licenses = [":truth_proto_license"], jars = [ "truth_proto/truth-liteproto-extension-1.0.1.jar", "truth_proto/truth-proto-extension-1.0.1.jar", ], - applicable_licenses = [":truth_proto_license"], ) license( @@ -798,9 +822,9 @@ license( distrib_java_import( name = "xz", + applicable_licenses = [":xz_license"], enable_distributions = ["debian"], jars = ["xz/xz-1.9.jar"], - applicable_licenses = [":xz_license"], ) license( diff --git a/third_party/auto/auto-value-gson-extension-1.3.1.jar b/third_party/auto/auto-value-gson-extension-1.3.1.jar new file mode 100644 index 00000000000000..b38408f16b2c3f Binary files /dev/null and b/third_party/auto/auto-value-gson-extension-1.3.1.jar differ diff --git a/third_party/auto/auto-value-gson-factory-1.3.1.jar b/third_party/auto/auto-value-gson-factory-1.3.1.jar new file mode 100644 index 00000000000000..b37a664c31598f Binary files /dev/null and b/third_party/auto/auto-value-gson-factory-1.3.1.jar differ diff --git a/third_party/auto/auto-value-gson-runtime-1.3.1.jar b/third_party/auto/auto-value-gson-runtime-1.3.1.jar new file mode 100644 index 00000000000000..bbd56d3880caef Binary files /dev/null and b/third_party/auto/auto-value-gson-runtime-1.3.1.jar differ diff --git a/third_party/auto/autotransient-1.0.0.jar b/third_party/auto/autotransient-1.0.0.jar new file mode 100644 index 00000000000000..3603e7be70e89f Binary files /dev/null and b/third_party/auto/autotransient-1.0.0.jar differ diff --git a/third_party/java/javapoet/BUILD b/third_party/java/javapoet/BUILD index 42bc23188676b6..9c368df3d961d9 100644 --- a/third_party/java/javapoet/BUILD +++ b/third_party/java/javapoet/BUILD @@ -16,5 +16,5 @@ filegroup( distrib_java_import( name = "javapoet", enable_distributions = ["debian"], - jars = ["javapoet-1.8.0.jar"], + jars = ["javapoet-1.12.1.jar"], ) diff --git a/third_party/java/javapoet/javapoet-1.12.1.jar b/third_party/java/javapoet/javapoet-1.12.1.jar new file mode 100644 index 00000000000000..9e1278e30516f1 Binary files /dev/null and b/third_party/java/javapoet/javapoet-1.12.1.jar differ diff --git a/third_party/java/javapoet/javapoet-1.8.0.jar b/third_party/java/javapoet/javapoet-1.8.0.jar deleted file mode 100644 index 6758b6d73920c4..00000000000000 Binary files a/third_party/java/javapoet/javapoet-1.8.0.jar and /dev/null differ