go: support //go:embed directives with embedsrcs attribute (#2806)

go_library, go_binary, and go_test now support the //go:embed
directive in Go 1.16. When building a package containing a //go:embed
directive, they'll match patterns against embedable files to generate
an embedcfg file to pass to the compiler.

Embeddable files must be listed in the new embedsrcs attribute.

The GoSource provider now has an embedsrcs field to support this.

Fixes #2775

go/core.rst

@@ -113,6 +113,14 @@ Attributes
 | files are also permitted. Files may be filtered at build time                                    |
 | using Go `build constraints`_.                                                                   |
 +----------------------------+-----------------------------+---------------------------------------+
+| :param:`embedsrcs`         | :type:`label_list`          | :value:`[]`                           |
++----------------------------+-----------------------------+---------------------------------------+
+| The list of files that may be embedded into the compiled package using                           |
+| ``//go:embed`` directives. All files must be in the same logical directory                       |
+| or a subdirectory as source files. All source files containing ``//go:embed``                    |
+| directives must be in the same logical directory. It's okay to mix static and                    |
+| generated source files and static and generated embeddable files.                                |
++----------------------------+-----------------------------+---------------------------------------+
 | :param:`x_defs`            | :type:`string_dict`         | :value:`{}`                           |
 +----------------------------+-----------------------------+---------------------------------------+
 | Map of defines to add to the go link command.                                                    |
@@ -309,6 +317,14 @@ Attributes
 | files are also permitted. Files may be filtered at build time                                    |
 | using Go `build constraints`_.                                                                   |
 +----------------------------+-----------------------------+---------------------------------------+
+| :param:`embedsrcs`         | :type:`label_list`          | :value:`[]`                           |
++----------------------------+-----------------------------+---------------------------------------+
+| The list of files that may be embedded into the compiled package using                           |
+| ``//go:embed`` directives. All files must be in the same logical directory                       |
+| or a subdirectory as source files. All source files containing ``//go:embed``                    |
+| directives must be in the same logical directory. It's okay to mix static and                    |
+| generated source files and static and generated embeddable files.                                |
++----------------------------+-----------------------------+---------------------------------------+
 | :param:`deps`              | :type:`label_list`          | :value:`[]`                           |
 +----------------------------+-----------------------------+---------------------------------------+
 | List of Go libraries this package imports directly.                                              |
@@ -547,6 +563,14 @@ Attributes
 | and the embedding library may not also have ``cgo = True``. See Embedding_                       |
 | for more information.                                                                            |
 +----------------------------+-----------------------------+---------------------------------------+
+| :param:`embedsrcs`         | :type:`label_list`          | :value:`[]`                           |
++----------------------------+-----------------------------+---------------------------------------+
+| The list of files that may be embedded into the compiled package using                           |
+| ``//go:embed`` directives. All files must be in the same logical directory                       |
+| or a subdirectory as source files. All source files containing ``//go:embed``                    |
+| directives must be in the same logical directory. It's okay to mix static and                    |
+| generated source files and static and generated embeddable files.                                |
++----------------------------+-----------------------------+---------------------------------------+
 | :param:`data`              | :type:`label_list`          | :value:`[]`                           |
 +----------------------------+-----------------------------+---------------------------------------+
 | List of files needed by this rule at run-time. This may include data files                       |

go/private/actions/archive.bzl

@@ -96,6 +96,7 @@ def emit_archive(go, source = None, _recompile_suffix = ""):
             go,
             sources = split.go + split.c + split.asm + split.cxx + split.objc + split.headers,
             cover = source.cover,
+            embedsrcs = source.embedsrcs,
             importpath = importpath,
             importmap = importmap,
             archives = direct,
@@ -119,6 +120,7 @@ def emit_archive(go, source = None, _recompile_suffix = ""):
             go,
             sources = split.go + split.c + split.asm + split.cxx + split.objc + split.headers,
             cover = source.cover,
+            embedsrcs = source.embedsrcs,
             importpath = importpath,
             importmap = importmap,
             archives = direct,
@@ -149,6 +151,7 @@ def emit_archive(go, source = None, _recompile_suffix = ""):
         orig_srcs = as_tuple(source.orig_srcs),
         _orig_src_map = tuple([source.orig_src_map.get(src, src) for src in source.srcs]),
         _cover = as_tuple(source.cover),
+        _embedsrcs = as_tuple(source.embedsrcs),
         _x_defs = tuple(source.x_defs.items()),
         _gc_goopts = as_tuple(source.gc_goopts),
         _cgo = source.cgo,

go/private/actions/compilepkg.bzl

@@ -34,6 +34,7 @@ def emit_compilepkg(
         go,
         sources = None,
         cover = None,
+        embedsrcs = [],
         importpath = "",
         importmap = "",
         archives = [],
@@ -56,14 +57,15 @@ def emit_compilepkg(
     if out_lib == None:
         fail("out_lib is a required parameter")
 
-    inputs = (sources + [go.package_list] +
+    inputs = (sources + embedsrcs + [go.package_list] +
               [archive.data.export_file for archive in archives] +
               go.sdk.tools + go.sdk.headers + go.stdlib.libs)
     outputs = [out_lib, out_export]
     env = go.env
 
     args = go.builder_args(go, "compilepkg")
     args.add_all(sources, before_each = "-src")
+    args.add_all(embedsrcs, before_each = "-embedsrc", expand_directories = False)
     if cover and go.coverdata:
         inputs.append(go.coverdata.data.export_file)
         args.add("-arc", _archive(go.coverdata))

go/private/context.bzl

@@ -170,6 +170,7 @@ def _merge_embed(source, embed):
     source["srcs"] = s.srcs + source["srcs"]
     source["orig_srcs"] = s.orig_srcs + source["orig_srcs"]
     source["orig_src_map"].update(s.orig_src_map)
+    source["embedsrcs"] = source["embedsrcs"] + s.embedsrcs
     source["cover"] = source["cover"] + s.cover
     source["deps"] = source["deps"] + s.deps
     source["x_defs"].update(s.x_defs)
@@ -214,13 +215,15 @@ def _library_to_source(go, attr, library, coverage_instrumented):
     attr_srcs = [f for t in getattr(attr, "srcs", []) for f in as_iterable(t.files)]
     generated_srcs = getattr(library, "srcs", [])
     srcs = attr_srcs + generated_srcs
+    embedsrcs = [f for t in getattr(attr, "embedsrcs", []) for f in as_iterable(t.files)]
     source = {
         "library": library,
         "mode": go.mode,
         "srcs": srcs,
         "orig_srcs": srcs,
         "orig_src_map": {},
         "cover": [],
+        "embedsrcs": embedsrcs,
         "x_defs": {},
         "deps": getattr(attr, "deps", []),
         "gc_goopts": getattr(attr, "gc_goopts", []),

go/private/rules/binary.bzl

@@ -88,6 +88,7 @@ _go_binary_kwargs = {
         "embed": attr.label_list(
             providers = [GoLibrary],
         ),
+        "embedsrcs": attr.label_list(allow_files = True),
         "importpath": attr.string(),
         "gc_goopts": attr.string_list(),
         "gc_linkopts": attr.string_list(),

go/private/rules/library.bzl

@@ -60,6 +60,7 @@ go_library = rule(
         "importmap": attr.string(),
         "importpath_aliases": attr.string_list(),  # experimental, undocumented
         "embed": attr.label_list(providers = [GoLibrary]),
+        "embedsrcs": attr.label_list(allow_files = True),
         "gc_goopts": attr.string_list(),
         "x_defs": attr.string_dict(),
         "cgo": attr.bool(),

go/private/rules/test.bzl

@@ -18,6 +18,7 @@ load(
 )
 load(
     "//go/private:common.bzl",
+    "as_list",
     "asm_exts",
     "cgo_exts",
     "go_exts",
@@ -182,6 +183,7 @@ _go_test_kwargs = {
         "srcs": attr.label_list(allow_files = go_exts + asm_exts + cgo_exts),
         "deps": attr.label_list(providers = [GoLibrary]),
         "embed": attr.label_list(providers = [GoLibrary]),
+        "embedsrcs": attr.label_list(allow_files = True),
         "importpath": attr.string(),
         "gc_goopts": attr.string_list(),
         "gc_linkopts": attr.string_list(),
@@ -391,21 +393,22 @@ def _recompile_external_deps(go, external_source, internal_archive, library_labe
         source = GoSource(
             library = library,
             mode = go.mode,
-            srcs = arc_data.srcs,
-            orig_srcs = arc_data.orig_srcs,
+            srcs = as_list(arc_data.srcs),
+            orig_srcs = as_list(arc_data.orig_srcs),
             orig_src_map = dict(zip(arc_data.srcs, arc_data._orig_src_map)),
             cover = arc_data._cover,
+            embedsrcs = as_list(arc_data._embedsrcs),
             x_defs = dict(arc_data._x_defs),
             deps = deps,
-            gc_goopts = arc_data._gc_goopts,
+            gc_goopts = as_list(arc_data._gc_goopts),
             runfiles = go._ctx.runfiles(files = arc_data.data_files),
             cgo = arc_data._cgo,
-            cdeps = arc_data._cdeps,
-            cppopts = arc_data._cppopts,
-            copts = arc_data._copts,
-            cxxopts = arc_data._cxxopts,
-            clinkopts = arc_data._clinkopts,
-            cgo_exports = arc_data._cgo_exports,
+            cdeps = as_list(arc_data._cdeps),
+            cppopts = as_list(arc_data._cppopts),
+            copts = as_list(arc_data._copts),
+            cxxopts = as_list(arc_data._cxxopts),
+            clinkopts = as_list(arc_data._clinkopts),
+            cgo_exports = as_list(arc_data._cgo_exports),
         )
 
         # If this archive needs to be recompiled, use go.archive.

go/providers.rst

@@ -139,6 +139,13 @@ method. In general, only rules_go should need to build or handle these.
 | Maps generated files in :param:`srcs` back to :param:`orig_srcs`. Not all                        |
 | generated files may appear in here.                                                              |
 +--------------------------------+-----------------------------------------------------------------+
+| :param:`embedsrcs`             | :type:`list of File`                                            |
++--------------------------------+-----------------------------------------------------------------+
+| Files that may be embedded into the compiled package using ``//go:embed``                        |
+| directives. All files must be in the same logical directory or a subdirectory                    |
+| as source files. However, it's okay to mix static and generated source files                     |
+| and static and generated embeddable files.                                                       |
++--------------------------------+-----------------------------------------------------------------+
 | :param:`cover`                 | :type:`list of File`                                            |
 +--------------------------------+-----------------------------------------------------------------+
 | List of source files to instrument for code coverage.                                            |

go/tools/builders/BUILD.bazel

@@ -7,6 +7,7 @@ go_test(
     srcs = [
         "filter.go",
         "filter_test.go",
+        "read.go",
     ],
 )
 
@@ -20,6 +21,7 @@ filegroup(
         "compile.go",
         "compilepkg.go",
         "cover.go",
+        "embedcfg.go",
         "env.go",
         "filter.go",
         "filter_buildid.go",
@@ -30,6 +32,7 @@ filegroup(
         "importcfg.go",
         "link.go",
         "pack.go",
+        "read.go",
         "replicate.go",
         "stdlib.go",
     ] + select({

go/tools/builders/asm.go

@@ -51,7 +51,7 @@ func asm(args []string) error {
 	source := flags.Args()[0]
 
 	// Filter the input file.
-	metadata, err := readFileInfo(build.Default, source, false)
+	metadata, err := readFileInfo(build.Default, source)
 	if err != nil {
 		return err
 	}

go/tools/builders/compilepkg.go

@@ -40,14 +40,15 @@ func compilePkg(args []string) error {
 
 	fs := flag.NewFlagSet("GoCompilePkg", flag.ExitOnError)
 	goenv := envFlags(fs)
-	var unfilteredSrcs, coverSrcs multiFlag
+	var unfilteredSrcs, coverSrcs, embedSrcs multiFlag
 	var deps archiveMultiFlag
 	var importPath, packagePath, nogoPath, packageListPath, coverMode string
 	var outPath, outFactsPath, cgoExportHPath string
 	var testFilter string
 	var gcFlags, asmFlags, cppFlags, cFlags, cxxFlags, objcFlags, objcxxFlags, ldFlags quoteMultiFlag
 	fs.Var(&unfilteredSrcs, "src", ".go, .c, .cc, .m, .mm, .s, or .S file to be filtered and compiled")
 	fs.Var(&coverSrcs, "cover", ".go file that should be instrumented for coverage (must also be a -src)")
+	fs.Var(&embedSrcs, "embedsrc", "file that may be compiled into the package with a //go:embed directive")
 	fs.Var(&deps, "arc", "Import path, package path, and file name of a direct dependency, separated by '='")
 	fs.StringVar(&importPath, "importpath", "", "The import path of the package being compiled. Not passed to the compiler, but may be displayed in debug data.")
 	fs.StringVar(&packagePath, "p", "", "The package path (importmap) of the package being compiled")
@@ -81,6 +82,9 @@ func compilePkg(args []string) error {
 	for i := range unfilteredSrcs {
 		unfilteredSrcs[i] = abs(unfilteredSrcs[i])
 	}
+	for i := range embedSrcs {
+		embedSrcs[i] = abs(embedSrcs[i])
+	}
 	for i := range coverSrcs {
 		coverSrcs[i] = abs(coverSrcs[i])
 	}
@@ -124,6 +128,7 @@ func compilePkg(args []string) error {
 		deps,
 		coverMode,
 		coverSrcs,
+		embedSrcs,
 		cgoEnabled,
 		cc,
 		gcFlags,
@@ -149,6 +154,7 @@ func compileArchive(
 	deps []archive,
 	coverMode string,
 	coverSrcs []string,
+	embedSrcs []string,
 	cgoEnabled bool,
 	cc string,
 	gcFlags []string,
@@ -318,6 +324,37 @@ func compileArchive(
 	}
 	defer os.Remove(importcfgPath)
 
+	// Build an embedcfg file mapping embed patterns to filenames.
+	// Embed patterns are relative to any one of a list of root directories
+	// that may contain embeddable files. Source files containing embed patterns
+	// must be in one of these root directories so the pattern appears to be
+	// relative to the source file. Usually, there are two roots: the source
+	// directory, and the output directory (so that generated files are
+	// embeddable). There may be additional roots if sources are in multiple
+	// directories (like if there are are generated source files).
+	var srcDirs []string
+	srcDirs = append(srcDirs, filepath.Dir(outPath))
+	for _, src := range srcs.goSrcs {
+		srcDirs = append(srcDirs, filepath.Dir(src.filename))
+	}
+	sort.Strings(srcDirs) // group duplicates to uniq them below.
+	embedRootDirs := srcDirs[:1]
+	for _, dir := range srcDirs {
+		prev := embedRootDirs[len(embedRootDirs)-1]
+		if dir == prev || strings.HasPrefix(dir, prev+string(filepath.Separator)) {
+			// Skip duplicates.
+			continue
+		}
+		embedRootDirs = append(embedRootDirs, dir)
+	}
+	embedcfgPath, err := buildEmbedcfgFile(srcs.goSrcs, embedSrcs, embedRootDirs, workDir)
+	if err != nil {
+		return err
+	}
+	if embedcfgPath != "" {
+		defer os.Remove(embedcfgPath)
+	}
+
 	// Run nogo concurrently.
 	var nogoChan chan error
 	outFactsPath := filepath.Join(workDir, nogoFact)
@@ -349,7 +386,7 @@ func compileArchive(
 	}
 
 	// Compile the filtered .go files.
-	if err := compileGo(goenv, goSrcs, packagePath, importcfgPath, asmHdrPath, symabisPath, gcFlags, outPath); err != nil {
+	if err := compileGo(goenv, goSrcs, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath, gcFlags, outPath); err != nil {
 		return err
 	}
 
@@ -419,9 +456,12 @@ func compileArchive(
 	return appendFiles(goenv, outXPath, []string{pkgDefPath})
 }
 
-func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, asmHdrPath, symabisPath string, gcFlags []string, outPath string) error {
+func compileGo(goenv *env, srcs []string, packagePath, importcfgPath, embedcfgPath, asmHdrPath, symabisPath string, gcFlags []string, outPath string) error {
 	args := goenv.goTool("compile")
 	args = append(args, "-p", packagePath, "-importcfg", importcfgPath, "-pack")
+	if embedcfgPath != "" {
+		args = append(args, "-embedcfg", embedcfgPath)
+	}
 	if asmHdrPath != "" {
 		args = append(args, "-asmhdr", asmHdrPath)
 	}