Lockfile cherry picks

.bazelci/postsubmit.yml

@@ -331,6 +331,10 @@ tasks:
       - "-//src/java_tools/import_deps_checker/..."
       # We hit connection timeout error when downloading multiple URLs on RBE, see b/217865760
       - "-//src/test/py/bazel:bazel_module_test"
+      - "-//src/test/py/bazel:bazel_lockfile_test"
+      - "-//src/test/py/bazel:bazel_overrides_test"
+      - "-//src/test/py/bazel:bazel_repo_mapping_test"
+      - "-//src/test/py/bazel:bazel_yanked_versions_test"
       - "-//src/test/shell/bazel:verify_workspace"
     include_json_profile:
       - build

.bazelci/presubmit.yml

@@ -321,6 +321,10 @@ tasks:
       - "-//src/java_tools/import_deps_checker/..."
       # We hit connection timeout error when downloading multiple URLs on RBE, see b/217865760
       - "-//src/test/py/bazel:bazel_module_test"
+      - "-//src/test/py/bazel:bazel_lockfile_test"
+      - "-//src/test/py/bazel:bazel_overrides_test"
+      - "-//src/test/py/bazel:bazel_repo_mapping_test"
+      - "-//src/test/py/bazel:bazel_yanked_versions_test"
       - "-//src/test/shell/bazel:verify_workspace"
   kythe_ubuntu2004:
     shell_commands:

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -25,7 +25,9 @@
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
+import com.google.devtools.build.lib.bazel.bzlmod.AttributeValues;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction;
+import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction;
@@ -46,6 +48,7 @@
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode;
+import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.RepositoryOverride;
 import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
 import com.google.devtools.build.lib.bazel.repository.downloader.DelegatingDownloader;
@@ -139,6 +142,7 @@ public class BazelRepositoryModule extends BlazeModule {
   private final AtomicBoolean ignoreDevDeps = new AtomicBoolean(false);
   private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING;
   private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR;
+  private LockfileMode bazelLockfileMode = LockfileMode.OFF;
   private List<String> allowedYankedVersions = ImmutableList.of();
   private SingleExtensionEvalFunction singleExtensionEvalFunction;
 
@@ -232,7 +236,8 @@ public void workspaceInit(
               public RepoSpec getRepoSpec(RepositoryName repoName) {
                 return RepoSpec.builder()
                     .setRuleClassName("local_config_platform")
-                    .setAttributes(ImmutableMap.of("name", repoName.getName()))
+                    .setAttributes(
+                        AttributeValues.create(ImmutableMap.of("name", repoName.getName())))
                     .build();
               }
 
@@ -249,7 +254,10 @@ public ResolutionReason getResolutionReason() {
         .addSkyFunction(
             SkyFunctions.MODULE_FILE,
             new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules))
-        .addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction())
+        .addSkyFunction(
+            SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(directories.getWorkspace()))
+        .addSkyFunction(
+            SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace()))
         .addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction())
         .addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction())
         .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction)
@@ -421,6 +429,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
       ignoreDevDeps.set(repoOptions.ignoreDevDependency);
       checkDirectDepsMode = repoOptions.checkDirectDependencies;
       bazelCompatibilityMode = repoOptions.bazelCompatibilityMode;
+      bazelLockfileMode = repoOptions.lockfileMode;
       allowedYankedVersions = repoOptions.allowedYankedVersions;
 
       if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) {
@@ -512,6 +521,7 @@ public ImmutableList<Injected> getPrecomputedValues() {
             BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, checkDirectDepsMode),
         PrecomputedValue.injected(
             BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, bazelCompatibilityMode),
+        PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode),
         PrecomputedValue.injected(
             BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, allowedYankedVersions));
   }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java

@@ -89,7 +89,7 @@ public RepoSpec build() {
     return RepoSpec.builder()
         .setBzlFile("@bazel_tools//tools/build_defs/repo:http.bzl")
         .setRuleClassName("http_archive")
-        .setAttributes(attrBuilder.buildOrThrow())
+        .setAttributes(AttributeValues.create(attrBuilder.buildOrThrow()))
         .build();
   }
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValues.java

@@ -0,0 +1,58 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package com.google.devtools.build.lib.bazel.bzlmod;
+
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.Maps;
+import com.ryanharter.auto.value.gson.GenerateTypeAdapter;
+import java.util.List;
+import java.util.Map;
+import net.starlark.java.eval.Dict;
+import net.starlark.java.eval.Starlark;
+
+/** Wraps a dictionary of attribute names and values. Always uses a dict to represent them */
+@AutoValue
+@GenerateTypeAdapter
+public abstract class AttributeValues {
+
+  public static AttributeValues create(Dict<String, Object> attribs) {
+    return new AutoValue_AttributeValues(attribs);
+  }
+
+  public static AttributeValues create(Map<String, Object> attribs) {
+    return new AutoValue_AttributeValues(
+        Dict.immutableCopyOf(Maps.transformValues(attribs, AttributeValues::valueToStarlark)));
+  }
+
+  public abstract Dict<String, Object> attributes();
+
+  // TODO(salmasamy) this is a copy of Attribute::valueToStarlark, Maybe think of a better place?
+  private static Object valueToStarlark(Object x) {
+    // Is x a non-empty string_list_dict?
+    if (x instanceof Map) {
+      Map<?, ?> map = (Map<?, ?>) x;
+      if (!map.isEmpty() && map.values().iterator().next() instanceof List) {
+        Dict.Builder<Object, Object> dict = Dict.builder();
+        for (Map.Entry<?, ?> e : map.entrySet()) {
+          dict.put(e.getKey(), Starlark.fromJava(e.getValue(), null));
+        }
+        return dict.buildImmutable();
+      }
+    }
+    // For all other attribute values, shallow conversion is safe.
+    return Starlark.fromJava(x, null);
+  }
+}

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValuesAdapter.java

@@ -0,0 +1,156 @@
+// Copyright 2023 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package com.google.devtools.build.lib.bazel.bzlmod;
+
+import com.google.devtools.build.lib.cmdline.Label;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+import com.google.gson.TypeAdapter;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import net.starlark.java.eval.Dict;
+import net.starlark.java.eval.EvalException;
+import net.starlark.java.eval.Mutability;
+import net.starlark.java.eval.Starlark;
+import net.starlark.java.eval.StarlarkInt;
+import net.starlark.java.eval.StarlarkList;
+
+/** Helps serialize/deserialize {@link AttributeValues}, which contains Starlark values. */
+public class AttributeValuesAdapter extends TypeAdapter<AttributeValues> {
+
+  @Override
+  public void write(JsonWriter out, AttributeValues attributeValues) throws IOException {
+    JsonObject jsonObject = new JsonObject();
+    for (Map.Entry<String, Object> entry : attributeValues.attributes().entrySet()) {
+      jsonObject.add(entry.getKey(), serializeObject(entry.getValue()));
+    }
+    out.jsonValue(jsonObject.toString());
+  }
+
+  @Override
+  public AttributeValues read(JsonReader in) throws IOException {
+    JsonObject jsonObject = JsonParser.parseReader(in).getAsJsonObject();
+    Dict.Builder<String, Object> dict = Dict.builder();
+    for (Map.Entry<String, JsonElement> entry : jsonObject.entrySet()) {
+      dict.put(entry.getKey(), deserializeObject(entry.getValue()));
+    }
+    return AttributeValues.create(dict.buildImmutable());
+  }
+
+  /**
+   * Starlark Object Types Bool Integer String Label List (Int, label, string) Dict (String,list) &
+   * (Label, String)
+   */
+  private JsonElement serializeObject(Object obj) {
+    if (obj.equals(Starlark.NONE)) {
+      return JsonNull.INSTANCE;
+    } else if (obj instanceof Boolean) {
+      return new JsonPrimitive((Boolean) obj);
+    } else if (obj instanceof StarlarkInt) {
+      try {
+        return new JsonPrimitive(((StarlarkInt) obj).toInt("serialization into the lockfile"));
+      } catch (EvalException e) {
+        throw new IllegalArgumentException("Unable to parse StarlarkInt to Integer: " + e);
+      }
+    } else if (obj instanceof String || obj instanceof Label) {
+      return new JsonPrimitive(serializeObjToString(obj));
+    } else if (obj instanceof Dict) {
+      JsonObject jsonObject = new JsonObject();
+      for (Map.Entry<?, ?> entry : ((Dict<?, ?>) obj).entrySet()) {
+        jsonObject.add(serializeObjToString(entry.getKey()), serializeObject(entry.getValue()));
+      }
+      return jsonObject;
+    } else if (obj instanceof StarlarkList) {
+      JsonArray jsonArray = new JsonArray();
+      for (Object item : (StarlarkList<?>) obj) {
+        jsonArray.add(serializeObject(item));
+      }
+      return jsonArray;
+    } else {
+      throw new IllegalArgumentException("Unsupported type: " + obj.getClass());
+    }
+  }
+
+  private Object deserializeObject(JsonElement json) {
+    if (json == null || json.isJsonNull()) {
+      return Starlark.NONE;
+    } else if (json.isJsonPrimitive()) {
+      JsonPrimitive jsonPrimitive = json.getAsJsonPrimitive();
+      if (jsonPrimitive.isBoolean()) {
+        return jsonPrimitive.getAsBoolean();
+      } else if (jsonPrimitive.isNumber()) {
+        return StarlarkInt.of(jsonPrimitive.getAsInt());
+      } else if (jsonPrimitive.isString()) {
+        return deserializeStringToObject(jsonPrimitive.getAsString());
+      } else {
+        throw new IllegalArgumentException("Unsupported JSON primitive: " + jsonPrimitive);
+      }
+    } else if (json.isJsonObject()) {
+      JsonObject jsonObject = json.getAsJsonObject();
+      Dict.Builder<Object, Object> dict = Dict.builder();
+      for (Map.Entry<String, JsonElement> entry : jsonObject.entrySet()) {
+        dict.put(deserializeStringToObject(entry.getKey()), deserializeObject(entry.getValue()));
+      }
+      return dict.buildImmutable();
+    } else if (json.isJsonArray()) {
+      JsonArray jsonArray = json.getAsJsonArray();
+      List<Object> list = new ArrayList<>();
+      for (JsonElement item : jsonArray) {
+        list.add(deserializeObject(item));
+      }
+      return StarlarkList.copyOf(Mutability.IMMUTABLE, list);
+    } else {
+      throw new IllegalArgumentException("Unsupported JSON element: " + json);
+    }
+  }
+
+  /**
+   * Serializes an object (Label or String) to String A label is converted to a String as it is. A
+   * String is being modified with a delimiter to be easily differentiated from the label when
+   * deserializing.
+   *
+   * @param obj String or Label
+   * @return serialized object
+   */
+  private String serializeObjToString(Object obj) {
+    if (obj instanceof Label) {
+      return ((Label) obj).getUnambiguousCanonicalForm();
+    }
+    return "--" + obj;
+  }
+
+  /**
+   * Deserializes a string to either a label or a String depending on the prefix. A string will have
+   * a delimiter at the start, else it is converted to a label.
+   *
+   * @param value String to be deserialized
+   * @return Object of type String of Label
+   */
+  private Object deserializeStringToObject(String value) {
+    if (value.startsWith("--")) {
+      return value.substring(2);
+    }
+    return Label.parseCanonicalUnchecked(value);
+  }
+}