Let bazel_deps replace Go deps

When a Go dependency is brought in both as a `bazel_dep` and as a
regular Go dependency via e.g. go.mod, the `bazel_dep` repo should be
used instead of creating a separate `go_repository`, which would risk
duplicate packages.

This is realized by registering every Bazel module with a
`go_deps.from_file` tag as providing the Go module of the same version
with the module path given in `go.mod`.

Gazelle is aware of these tags and generates either a canonical label or
 a label using the apparent module repo name (the latter only for the
root module and only if it directly depends on the `bazel_dep`).

cmd/gazelle/fix-update.go

@@ -156,10 +156,22 @@ func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) erro
 			GoPrefix: imp,
 		})
 	}
+
+	moduleToApparentName, err := module.ExtractModuleToApparentNameMapping(c.RepoRoot)
+	if err != nil {
+		return err
+	}
+
 	for _, r := range c.Repos {
 		if r.Kind() == "go_repository" {
+			var name string
+			if apparentName := moduleToApparentName(r.AttrString("module_name")); apparentName != "" {
+				name = apparentName
+			} else {
+				name = r.Name()
+			}
 			uc.repos = append(uc.repos, repo.Repo{
-				Name:     r.Name(),
+				Name:     name,
 				GoPrefix: r.AttrString("importpath"),
 			})
 		}

config/config.go

@@ -104,6 +104,9 @@ type Config struct {
 	// extensions as well. Values in here may be populated by command line
 	// arguments, directives in build files, or other mechanisms.
 	Exts map[string]interface{}
+
+	// Whether Gazelle is loaded as a Bzlmod 'bazel_dep'.
+	Bzlmod bool
 }
 
 // MappedKind describes a replacement to use for a built-in kind.
@@ -192,6 +195,7 @@ type CommonConfigurer struct {
 	repoRoot, buildFileNames, readBuildFilesDir, writeBuildFilesDir string
 	indexLibraries, strict                                          bool
 	langCsv                                                         string
+	bzlmod                                                          bool
 }
 
 func (cc *CommonConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *Config) {
@@ -202,6 +206,7 @@ func (cc *CommonConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *Confi
 	fs.StringVar(&cc.readBuildFilesDir, "experimental_read_build_files_dir", "", "path to a directory where build files should be read from (instead of -repo_root)")
 	fs.StringVar(&cc.writeBuildFilesDir, "experimental_write_build_files_dir", "", "path to a directory where build files should be written to (instead of -repo_root)")
 	fs.StringVar(&cc.langCsv, "lang", "", "if non-empty, process only these languages (e.g. \"go,proto\")")
+	fs.BoolVar(&cc.bzlmod, "bzlmod", false, "for internal usage only")
 }
 
 func (cc *CommonConfigurer) CheckFlags(fs *flag.FlagSet, c *Config) error {
@@ -244,6 +249,7 @@ func (cc *CommonConfigurer) CheckFlags(fs *flag.FlagSet, c *Config) error {
 	if len(cc.langCsv) > 0 {
 		c.Langs = strings.Split(cc.langCsv, ",")
 	}
+	c.Bzlmod = cc.bzlmod
 	return nil
 }
 

def.bzl

@@ -73,6 +73,8 @@ def _gazelle_runner_impl(ctx):
         args.extend(["-go_prefix", ctx.attr.prefix])
     if ctx.attr.build_tags:
         args.extend(["-build_tags", ",".join(ctx.attr.build_tags)])
+    if IS_BZLMOD_ENABLED:
+        args.append("-bzlmod")
     args.extend([ctx.expand_location(arg, ctx.attr.data) for arg in ctx.attr.extra_args])
 
     for key in ctx.attr.env:

internal/bzlmod/go_deps.bzl

@@ -18,12 +18,7 @@ load(
 
 visibility("//")
 
-# These Go modules are imported as Bazel modules via bazel_dep, not as
-# go_repository.
-_IGNORED_MODULE_PATHS = [
-    "github.com/bazelbuild/bazel-gazelle",
-    "github.com/bazelbuild/rules_go",
-]
+_HIGHEST_VERSION_SENTINEL = semver.to_comparable("999999.999999.999999")
 
 _FORBIDDEN_OVERRIDE_TAG = """\
 Using the "go_deps.{tag_class}" tag in a non-root Bazel module is forbidden, \
@@ -168,6 +163,7 @@ def _go_repository_config_impl(ctx):
             "go_repository",
             name = name,
             importpath = importpath,
+            module_name = ctx.attr.module_names.get(name),
             build_naming_convention = ctx.attr.build_naming_conventions.get(name),
         ))
 
@@ -178,6 +174,7 @@ _go_repository_config = repository_rule(
     implementation = _go_repository_config_impl,
     attrs = {
         "importpaths": attr.string_dict(mandatory = True),
+        "module_names": attr.string_dict(mandatory = True),
         "build_naming_conventions": attr.string_dict(mandatory = True),
     },
 )
@@ -189,6 +186,7 @@ def _go_deps_impl(module_ctx):
     module_resolutions = {}
     sums = {}
     replace_map = {}
+    bazel_deps = {}
 
     gazelle_overrides = {}
     module_overrides = {}
@@ -245,7 +243,7 @@ def _go_deps_impl(module_ctx):
             )
         additional_module_tags = []
         for from_file_tag in module.tags.from_file:
-            module_tags_from_go_mod, go_mod_replace_map = deps_from_go_mod(module_ctx, from_file_tag.go_mod)
+            module_path, module_tags_from_go_mod, go_mod_replace_map = deps_from_go_mod(module_ctx, from_file_tag.go_mod)
             is_dev_dependency = _is_dev_dependency(module_ctx, from_file_tag)
             additional_module_tags += [
                 with_replaced_or_new_fields(tag, _is_dev_dependency = is_dev_dependency)
@@ -254,6 +252,21 @@ def _go_deps_impl(module_ctx):
 
             if module.is_root:
                 replace_map.update(go_mod_replace_map)
+            else:
+                # Register this Bazel module as providing the specified Go module. It participates
+                # in version resolution using its registry version, which is assumed to be an
+                # actual semver. An empty version string signals an override, which is assumed to
+                # be newer than any other version.
+                # TODO: Decide whether and how to handle non-semver versions.
+                raw_version = _canonicalize_raw_version(module.version)
+                version = semver.to_comparable(raw_version) if raw_version else _HIGHEST_VERSION_SENTINEL
+                if module_path not in bazel_deps or version > bazel_deps[module_path].version:
+                    bazel_deps[module_path] = struct(
+                        module_name = module.name,
+                        repo_name = "@" + from_file_tag.go_mod.workspace_name,
+                        version = version,
+                        raw_version = raw_version,
+                    )
 
             # Load all sums from transitively resolved `go.sum` files that have modules.
             if len(module_tags_from_go_mod) > 0:
@@ -283,7 +296,7 @@ def _go_deps_impl(module_ctx):
         for module_tag in module.tags.module + additional_module_tags:
             if module_tag.path in paths:
                 fail("Duplicate Go module path \"{}\" in module \"{}\".".format(module_tag.path, module.name))
-            if module_tag.path in _IGNORED_MODULE_PATHS:
+            if module_tag.path in bazel_deps:
                 continue
             paths[module_tag.path] = None
             raw_version = _canonicalize_raw_version(module_tag.version)
@@ -339,7 +352,23 @@ def _go_deps_impl(module_ctx):
                     # not ever report an implicit version upgrade.
                     root_versions.pop(path)
                 else:
-                    root_versions[path] = new_version
+                    root_versions[path] = replace.version
+
+    bazel_deps_to_use = {}
+    for path, bazel_dep in bazel_deps.items():
+        # We can't apply overrides to Bazel dependencies and thus fall back to using the Go module.
+        if path in gazelle_overrides or path in module_overrides or path in replace_map:
+            continue
+
+        # Only use the Bazel module if it is at least as the high as the required Go module version.
+        if path in module_resolutions and bazel_dep.version < module_resolutions[path].version:
+            continue
+
+        # The Bazel module may not be declared as a Go dependency, so don't fail if it doesn't
+        # appear in these maps.
+        module_resolutions.pop(path, None)
+        root_versions.pop(path, None)
+        bazel_deps_to_use[path] = None
 
     for path, root_version in root_versions.items():
         if semver.to_comparable(root_version) < module_resolutions[path].version:
@@ -374,6 +403,15 @@ def _go_deps_impl(module_ctx):
         importpaths = {
             module.repo_name: path
             for path, module in module_resolutions.items()
+        } | {
+            info.repo_name: path
+            for path, info in bazel_deps.items()
+            if path in bazel_deps_to_use
+        },
+        module_names = {
+            info.repo_name: info.module_name
+            for path, info in bazel_deps.items()
+            if path in bazel_deps_to_use
         },
         build_naming_conventions = drop_nones({
             module.repo_name: get_directive_value(

internal/bzlmod/go_mod.bzl

@@ -11,8 +11,8 @@ def deps_from_go_mod(module_ctx, go_mod_label):
         go_mod_label: a Label for a `go.mod` file.
 
     Returns:
-        A list of structs is returned. Each struct represents a `require` statement
-        from the go.mod file.
+        a tuple (Go module path, deps, replace map), where deps is a list of structs representing
+        `require` statements from the go.mod file.
     """
     _check_go_mod_name(go_mod_label.name)
 
@@ -33,7 +33,7 @@ def deps_from_go_mod(module_ctx, go_mod_label):
             indirect = require.indirect,
         ))
 
-    return deps, go_mod.replace_map
+    return go_mod.module, deps, go_mod.replace_map
 
 def parse_go_mod(content, path):
     # See https://go.dev/ref/mod#go-mod-file.

internal/go_repository.bzl

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("//internal:common.bzl", "env_execute", "executable_extension")
+load("//internal:common.bzl", "IS_BZLMOD_ENABLED", "env_execute", "executable_extension")
 load("//internal:go_repository_cache.bzl", "read_cache_env")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "patch", "read_user_netrc", "use_netrc")
 
@@ -304,6 +304,8 @@ def _go_repository_impl(ctx):
             cmd.extend(["-proto", ctx.attr.build_file_proto_mode])
         if ctx.attr.build_naming_convention:
             cmd.extend(["-go_naming_convention", ctx.attr.build_naming_convention])
+        if IS_BZLMOD_ENABLED:
+            cmd.append("-bzlmod")
         cmd.extend(ctx.attr.build_extra_args)
         cmd.append(ctx.path(""))
         ctx.report_progress("running Gazelle")

language/go/resolve.go

@@ -145,12 +145,14 @@ func ResolveGo(c *config.Config, ix *resolve.RuleIndex, rc *repo.RemoteCache, im
 	// These have names that don't following conventions and they're
 	// typeically declared with http_archive, not go_repository, so Gazelle
 	// won't recognize them.
-	if pathtools.HasPrefix(imp, "github.com/bazelbuild/rules_go") {
-		pkg := pathtools.TrimPrefix(imp, "github.com/bazelbuild/rules_go")
-		return label.New("io_bazel_rules_go", pkg, "go_default_library"), nil
-	} else if pathtools.HasPrefix(imp, "github.com/bazelbuild/bazel-gazelle") {
-		pkg := pathtools.TrimPrefix(imp, "github.com/bazelbuild/bazel-gazelle")
-		return label.New("bazel_gazelle", pkg, "go_default_library"), nil
+	if !c.Bzlmod {
+		if pathtools.HasPrefix(imp, "github.com/bazelbuild/rules_go") {
+			pkg := pathtools.TrimPrefix(imp, "github.com/bazelbuild/rules_go")
+			return label.New("io_bazel_rules_go", pkg, "go_default_library"), nil
+		} else if pathtools.HasPrefix(imp, "github.com/bazelbuild/bazel-gazelle") {
+			pkg := pathtools.TrimPrefix(imp, "github.com/bazelbuild/bazel-gazelle")
+			return label.New("bazel_gazelle", pkg, "go_default_library"), nil
+		}
 	}
 
 	if !c.IndexLibraries {

tests/bcr/MODULE.bazel

@@ -8,7 +8,7 @@ local_path_override(
     path = "../..",
 )
 
-bazel_dep(name = "rules_go", version = "0.39.1")
+bazel_dep(name = "rules_go", version = "0.39.1", repo_name = "my_rules_go")
 
 go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
 
@@ -66,9 +66,12 @@ use_repo(
     go_deps,
     "com_github_bmatcuk_doublestar_v4",
     "com_github_datadog_sketches_go",
+    "com_github_fmeum_dep_on_gazelle",
     "com_github_google_safetext",
     "com_github_pelletier_go_toml",
     "com_github_stretchr_testify",
     # It is not necessary to list transitive dependencies here, only direct ones.
     # "in_gopkg_yaml_v3",
+    # Only used for testing.
+    "bazel_gazelle_go_repository_config",
 )

tests/bcr/go.mod

@@ -7,7 +7,16 @@ go 1.19
 replace github.com/bmatcuk/doublestar/v4 => github.com/bmatcuk/doublestar v1.3.4
 
 require (
-	github.com/bmatcuk/doublestar/v4 v4.0.0
-	github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2
 	github.com/DataDog/sketches-go v1.4.1
+	github.com/bazelbuild/rules_go v0.39.1
+	github.com/bmatcuk/doublestar/v4 v4.6.0
+	github.com/fmeum/dep_on_gazelle v1.0.0
+	github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2
+)
+
+require (
+	github.com/bazelbuild/bazel-gazelle v0.30.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 )

tests/bcr/go.sum

@@ -1,20 +1,42 @@
 github.com/DataDog/sketches-go v1.4.1 h1:j5G6as+9FASM2qC36lvpvQAj9qsv/jUs3FtO8CwZNAY=
 github.com/DataDog/sketches-go v1.4.1/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk=
+github.com/bazelbuild/bazel-gazelle v0.30.0 h1:q9XLWQSCA5NZPJ98WFqicHkq6fxrDPnfvMO1XycQBMg=
+github.com/bazelbuild/bazel-gazelle v0.30.0/go.mod h1:6RxhjM1v/lTpD3JlMpKUCcdus4tvdqsqdfbjYi+czYs=
+github.com/bazelbuild/rules_go v0.39.1 h1:wkJLUDx59dntWMghuL8++GteoU1To6sRoKJXuyFtmf8=
+github.com/bazelbuild/rules_go v0.39.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
 github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
 github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fmeum/dep_on_gazelle v1.0.0 h1:7gEtQ2CoD77tYca+1iUnKjIBUZ4mX7mZwjdWp3uuN/E=
+github.com/fmeum/dep_on_gazelle v1.0.0/go.mod h1:VYCjwfsyRHOJL8oenaEjhIzgM7Oj70iTxgJ2RfXbYr0=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 h1:SJ+NtwL6QaZ21U+IrK7d0gGgpjGGvd2kz+FzTHVzdqI=
 github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC9t8wNnpPdctvtSUOPUUg4SHeE6vR1Ir2hmg=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
+github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tests/bcr/pkg/BUILD.bazel

@@ -1,13 +1,16 @@
-load("@rules_go//go:def.bzl", "go_test")
+load("@my_rules_go//go:def.bzl", "go_test")
 
 go_test(
     name = "pkg_test",
-    srcs = ["mvs_test.go"],
+    srcs = ["pkg_test.go"],
     deps = [
+        "//pkg/data",
         "@com_github_bmatcuk_doublestar_v4//:doublestar",
         "@com_github_datadog_sketches_go//ddsketch",
+        "@com_github_fmeum_dep_on_gazelle//:dep_on_gazelle",
         "@com_github_google_safetext//yamltemplate",
         "@com_github_pelletier_go_toml//:go-toml",
         "@com_github_stretchr_testify//require:go_default_library",
+        "@my_rules_go//go/runfiles",
     ],
 )

tests/bcr/pkg/data/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@my_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "data",
+    srcs = ["data.go"],
+    data = ["@bazel_gazelle_go_repository_config//:WORKSPACE"],
+    importpath = "github.com/bazelbuild/bazel-gazelle/tests/bcr/pkg/data",
+    visibility = ["//visibility:public"],
+)

tests/bcr/pkg/data/data.go

@@ -0,0 +1,3 @@
+package data
+
+const RepoConfigRlocationPath = "bazel_gazelle_go_repository_config/WORKSPACE"