gopls/internal/lsp: add semantic highlighting for go: directives

Highlight known go:directives in their surrounding comment.
Unknown or misspelled directives are skipped to signal to the
end user that something is off.

Closes golang/go#63538

Change-Id: Idf926106600b734ce2076366798acef0051181d6
Reviewed-on: https://go-review.googlesource.com/c/tools/+/536915
Auto-Submit: Robert Findley <rfindley@google.com>
Reviewed-by: Robert Findley <rfindley@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Hyang-Ah Hana Kim <hyangah@gmail.com>

gopls/internal/lsp/semantic.go

@@ -143,6 +143,10 @@ func (e *encoded) semantics() {
 	}
 	for _, cg := range f.Comments {
 		for _, c := range cg.List {
+			if strings.HasPrefix(c.Text, "//go:") {
+				e.godirective(c)
+				continue
+			}
 			if !strings.Contains(c.Text, "\n") {
 				e.token(c.Pos(), len(c.Text), tokComment, nil)
 				continue
@@ -997,3 +1001,52 @@ var (
 		"deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary",
 	}
 )
+
+var godirectives = map[string]struct{}{
+	// https://pkg.go.dev/cmd/compile
+	"noescape":       {},
+	"uintptrescapes": {},
+	"noinline":       {},
+	"norace":         {},
+	"nosplit":        {},
+	"linkname":       {},
+
+	// https://pkg.go.dev/go/build
+	"build":               {},
+	"binary-only-package": {},
+	"embed":               {},
+}
+
+// Tokenize godirective at the start of the comment c, if any, and the surrounding comment.
+// If there is any failure, emits the entire comment as a tokComment token.
+// Directives are highlighted as-is, even if used incorrectly. Typically there are
+// dedicated analyzers that will warn about misuse.
+func (e *encoded) godirective(c *ast.Comment) {
+	// First check if '//go:directive args...' is a valid directive.
+	directive, args, _ := strings.Cut(c.Text, " ")
+	kind, _ := stringsCutPrefix(directive, "//go:")
+	if _, ok := godirectives[kind]; !ok {
+		// Unknown go: directive.
+		e.token(c.Pos(), len(c.Text), tokComment, nil)
+		return
+	}
+
+	// Make the 'go:directive' part stand out, the rest is comments.
+	e.token(c.Pos(), len("//"), tokComment, nil)
+
+	directiveStart := c.Pos() + token.Pos(len("//"))
+	e.token(directiveStart, len(directive[len("//"):]), tokNamespace, nil)
+
+	if len(args) > 0 {
+		tailStart := c.Pos() + token.Pos(len(directive)+len(" "))
+		e.token(tailStart, len(args), tokComment, nil)
+	}
+}
+
+// Go 1.20 strings.CutPrefix.
+func stringsCutPrefix(s, prefix string) (after string, found bool) {
+	if !strings.HasPrefix(s, prefix) {
+		return s, false
+	}
+	return s[len(prefix):], true
+}

gopls/internal/regtest/misc/semantictokens_test.go

@@ -145,3 +145,60 @@ func New[K int, V any]() Smap[K, V] {
 		}
 	})
 }
+
+func TestSemanticGoDirectives(t *testing.T) {
+	src := `
+-- go.mod --
+module example.com
+
+go 1.19
+-- main.go --
+package foo
+
+//go:linkname now time.Now
+func now()
+
+//go:noinline
+func foo() {}
+
+// Mentioning go:noinline should not tokenize.
+
+//go:notadirective
+func bar() {}
+`
+	want := []fake.SemanticToken{
+		{Token: "package", TokenType: "keyword"},
+		{Token: "foo", TokenType: "namespace"},
+
+		{Token: "//", TokenType: "comment"},
+		{Token: "go:linkname", TokenType: "namespace"},
+		{Token: "now time.Now", TokenType: "comment"},
+		{Token: "func", TokenType: "keyword"},
+		{Token: "now", TokenType: "function", Mod: "definition"},
+
+		{Token: "//", TokenType: "comment"},
+		{Token: "go:noinline", TokenType: "namespace"},
+		{Token: "func", TokenType: "keyword"},
+		{Token: "foo", TokenType: "function", Mod: "definition"},
+
+		{Token: "// Mentioning go:noinline should not tokenize.", TokenType: "comment"},
+
+		{Token: "//go:notadirective", TokenType: "comment"},
+		{Token: "func", TokenType: "keyword"},
+		{Token: "bar", TokenType: "function", Mod: "definition"},
+	}
+
+	WithOptions(
+		Modes(Default),
+		Settings{"semanticTokens": true},
+	).Run(t, src, func(t *testing.T, env *Env) {
+		env.OpenFile("main.go")
+		seen, err := env.Editor.SemanticTokens(env.Ctx, "main.go")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if x := cmp.Diff(want, seen); x != "" {
+			t.Errorf("Semantic tokens do not match (-want +got):\n%s", x)
+		}
+	})
+}