all: merge master (c353b05) into gopls-release-branch.0.9

Add replace directive for x/tools to gopls/go.mod.

For golang/go#53412

Merge List:

+ 2022-06-16 c353b05 internal/lsp/cache: delete checkSnapshotLocked
+ 2022-06-16 567c98b internal/lsp/cache: don't walk URIs to invalidate metadata
+ 2022-06-16 dffd645 internal/lsp/cache: only clone metadata if something changed
+ 2022-06-16 4ba3d22 internal/lsp/cache: clone the metadata graph when clearing ShouldLoad
+ 2022-06-16 39d3d49 internal/lsp/cache: use metadataGraph.Clone in snapshot.clone
+ 2022-06-16 8a92078 internal/lsp/cache: build a new metadata graph on load
+ 2022-06-16 9f38ef7 internal/lsp/cache: derive workspace packages from metadata
+ 2022-06-16 041035c Revert "internal/lsp/cache: reduce critical sections"
+ 2022-06-16 d097bc9 gopls/internal/vulncheck: include nonaffecting vulnerability info
+ 2022-06-16 e8b9ff1 gopls/internal/govulncheck: sync x/vuln@4eb5ba4
+ 2022-06-15 654a14b internal/lsp/cache: reduce critical sections
+ 2022-06-14 27db7f4 gopls: update golang.org/x/vuln to latest @4eb5ba4
+ 2022-06-14 c993be6 go/analysis/internal/checker: log codeFact error, remove unused action.inputs
+ 2022-06-14 ed27611 internal/lsp/cache: cache known subdirs pattern
+ 2022-06-14 ebc084a internal/lsp: add inlay hints count to test summary
+ 2022-06-13 c15c045 internal/lsp: enable inlay hint tests
+ 2022-06-13 0343989 internal/lsp: fix error message for inlay hints
+ 2022-06-13 a41fc98 internal/lsp/cache: use [256]byte Hash instead of hex digit string
+ 2022-06-10 c41ddce internal/lsp: include padding in inlay hint marker tests
+ 2022-06-10 5e48d26 internal/lsp: add inlay hints for composite literal names
+ 2022-06-10 83b0675 internal/lsp: add inlay hints for constant values
+ 2022-06-10 ecc1479 internal/lsp: add inlay hints for variable types
+ 2022-06-10 65c0181 internal/lsp: support textDocument/inlayHint for parameter names

Change-Id: Idc9bcd1253bbaa32a601867f25472645b104dd05

go/analysis/internal/checker/checker.go

@@ -578,7 +578,6 @@ type action struct {
 	deps         []*action
 	objectFacts  map[objectFactKey]analysis.Fact
 	packageFacts map[packageFactKey]analysis.Fact
-	inputs       map[*analysis.Analyzer]interface{}
 	result       interface{}
 	diagnostics  []analysis.Diagnostic
 	err          error
@@ -766,7 +765,7 @@ func inheritFacts(act, dep *action) {
 		if serialize {
 			encodedFact, err := codeFact(fact)
 			if err != nil {
-				log.Panicf("internal error: encoding of %T fact failed in %v", fact, act)
+				log.Panicf("internal error: encoding of %T fact failed in %v: %v", fact, act, err)
 			}
 			fact = encodedFact
 		}
@@ -894,7 +893,7 @@ func (act *action) exportObjectFact(obj types.Object, fact analysis.Fact) {
 func (act *action) allObjectFacts() []analysis.ObjectFact {
 	facts := make([]analysis.ObjectFact, 0, len(act.objectFacts))
 	for k := range act.objectFacts {
-		facts = append(facts, analysis.ObjectFact{k.obj, act.objectFacts[k]})
+		facts = append(facts, analysis.ObjectFact{Object: k.obj, Fact: act.objectFacts[k]})
 	}
 	return facts
 }
@@ -940,7 +939,7 @@ func factType(fact analysis.Fact) reflect.Type {
 func (act *action) allPackageFacts() []analysis.PackageFact {
 	facts := make([]analysis.PackageFact, 0, len(act.packageFacts))
 	for k := range act.packageFacts {
-		facts = append(facts, analysis.PackageFact{k.pkg, act.packageFacts[k]})
+		facts = append(facts, analysis.PackageFact{Package: k.pkg, Fact: act.packageFacts[k]})
 	}
 	return facts
 }

gopls/go.mod

@@ -9,8 +9,8 @@ require (
 	github.com/sergi/go-diff v1.1.0
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4
 	golang.org/x/sys v0.0.0-20220209214540-3681064d5158
-	golang.org/x/tools v0.1.11-0.20220330174940-8e193c2ba95e
-	golang.org/x/vuln v0.0.0-20220503210553-a5481fb0c8be
+	golang.org/x/tools v0.1.11-0.20220523181440-ccb10502d1a5
+	golang.org/x/vuln v0.0.0-20220613164644-4eb5ba49563c
 	honnef.co/go/tools v0.3.0
 	mvdan.cc/gofumpt v0.3.0
 	mvdan.cc/xurls/v2 v2.4.0

gopls/go.sum

@@ -73,6 +73,8 @@ golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/vuln v0.0.0-20220503210553-a5481fb0c8be h1:jokAF1mfylAi1iTQx7C44B7vyXUcSEMw8eDv0PzNu8s=
 golang.org/x/vuln v0.0.0-20220503210553-a5481fb0c8be/go.mod h1:twca1SxmF6/i2wHY/mj1vLIkkHdp+nil/yA32ZOP4kg=
+golang.org/x/vuln v0.0.0-20220613164644-4eb5ba49563c h1:r5bbIROBQtRRgoutV8Q3sFY58VzHW6jMBYl48ANSyS4=
+golang.org/x/vuln v0.0.0-20220613164644-4eb5ba49563c/go.mod h1:UZshlUPxXeGUM9I14UOawXQg6yosDE9cr1vKY/DzgWo=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=

gopls/internal/govulncheck/source.go

@@ -59,6 +59,8 @@ func LoadPackages(cfg *packages.Config, patterns ...string) ([]*vulncheck.Packag
 
 // Source calls vulncheck.Source on the Go source in pkgs. It returns the result
 // with Vulns trimmed to those that are actually called.
+//
+// This function is being used by the Go IDE team.
 func Source(ctx context.Context, pkgs []*vulncheck.Package, c client.Client) (*vulncheck.Result, error) {
 	r, err := vulncheck.Source(ctx, pkgs, &vulncheck.Config{Client: c})
 	if err != nil {
@@ -77,14 +79,21 @@ func Source(ctx context.Context, pkgs []*vulncheck.Package, c client.Client) (*v
 
 // CallInfo is information about calls to vulnerable functions.
 type CallInfo struct {
-	CallStacks     map[*vulncheck.Vuln][]vulncheck.CallStack // all call stacks
-	VulnGroups     [][]*vulncheck.Vuln                       // vulns grouped by ID and package
-	ModuleVersions map[string]string                         // map from module paths to versions
-	TopPackages    map[string]bool                           // top-level packages
+	// CallStacks contains all call stacks to vulnerable functions.
+	CallStacks map[*vulncheck.Vuln][]vulncheck.CallStack
+
+	// VulnGroups contains vulnerabilities grouped by ID and package.
+	VulnGroups [][]*vulncheck.Vuln
+
+	// ModuleVersions is a map of module paths to versions.
+	ModuleVersions map[string]string
+
+	// TopPackages contains the top-level packages in the call info.
+	TopPackages map[string]bool
 }
 
 // GetCallInfo computes call stacks and related information from a vulncheck.Result.
-// I also makes a set of top-level packages from pkgs.
+// It also makes a set of top-level packages from pkgs.
 func GetCallInfo(r *vulncheck.Result, pkgs []*vulncheck.Package) *CallInfo {
 	pset := map[string]bool{}
 	for _, p := range pkgs {

gopls/internal/regtest/bench/bench_test.go

@@ -8,6 +8,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"runtime"
 	"runtime/pprof"
 	"testing"
 
@@ -66,6 +67,7 @@ func TestBenchmarkIWL(t *testing.T) {
 	results := testing.Benchmark(func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
 			WithOptions(opts...).Run(t, "", func(t *testing.T, env *Env) {})
+
 		}
 	})
 
@@ -192,3 +194,31 @@ func TestBenchmarkDidChange(t *testing.T) {
 		printBenchmarkResults(result)
 	})
 }
+
+// TestPrintMemStats measures the memory usage of loading a project.
+// It uses the same -didchange_dir flag as above.
+// Always run it in isolation since it measures global heap usage.
+//
+// Kubernetes example:
+//   $ go test -run=TestPrintMemStats -didchange_dir=$HOME/w/kubernetes
+//   TotalAlloc:      5766 MB
+//   HeapAlloc:       1984 MB
+//
+// Both figures exhibit variance of less than 1%.
+func TestPrintMemStats(t *testing.T) {
+	if *benchDir == "" {
+		t.Skip("-didchange_dir is not set")
+	}
+
+	// Load the program...
+	opts := benchmarkOptions(*benchDir)
+	WithOptions(opts...).Run(t, "", func(_ *testing.T, env *Env) {
+		// ...and print the memory usage.
+		runtime.GC()
+		runtime.GC()
+		var mem runtime.MemStats
+		runtime.ReadMemStats(&mem)
+		t.Logf("TotalAlloc:\t%d MB", mem.TotalAlloc/1e6)
+		t.Logf("HeapAlloc:\t%d MB", mem.HeapAlloc/1e6)
+	})
+}

gopls/internal/vulncheck/command.go

@@ -11,12 +11,15 @@ import (
 	"context"
 	"log"
 	"os"
+	"sort"
 	"strings"
 
 	"golang.org/x/tools/go/packages"
 	gvc "golang.org/x/tools/gopls/internal/govulncheck"
 	"golang.org/x/tools/internal/lsp/command"
 	"golang.org/x/vuln/client"
+	"golang.org/x/vuln/osv"
+	"golang.org/x/vuln/vulncheck"
 )
 
 func init() {
@@ -79,29 +82,84 @@ func (c *cmd) Run(ctx context.Context, cfg *packages.Config, patterns ...string)
 	}
 	log.Printf("loaded %d packages\n", len(loadedPkgs))
 
-	r, err := gvc.Source(ctx, loadedPkgs, c.Client)
+	log.Printf("analyzing %d packages...\n", len(loadedPkgs))
+
+	r, err := vulncheck.Source(ctx, loadedPkgs, &vulncheck.Config{Client: c.Client})
 	if err != nil {
 		return nil, err
 	}
+	unaffectedMods := filterUnaffected(r.Vulns)
+	r.Vulns = filterCalled(r)
+
 	callInfo := gvc.GetCallInfo(r, loadedPkgs)
-	return toVulns(callInfo)
+	return toVulns(callInfo, unaffectedMods)
 	// TODO: add import graphs.
 }
 
-func toVulns(ci *gvc.CallInfo) ([]Vuln, error) {
+// filterCalled returns vulnerabilities where the symbols are actually called.
+func filterCalled(r *vulncheck.Result) []*vulncheck.Vuln {
+	var vulns []*vulncheck.Vuln
+	for _, v := range r.Vulns {
+		if v.CallSink != 0 {
+			vulns = append(vulns, v)
+		}
+	}
+	return vulns
+}
+
+// filterUnaffected returns vulnerabilities where no symbols are called,
+// grouped by module.
+func filterUnaffected(vulns []*vulncheck.Vuln) map[string][]*osv.Entry {
+	// It is possible that the same vuln.OSV.ID has vuln.CallSink != 0
+	// for one symbol, but vuln.CallSink == 0 for a different one, so
+	// we need to filter out ones that have been called.
+	called := map[string]bool{}
+	for _, vuln := range vulns {
+		if vuln.CallSink != 0 {
+			called[vuln.OSV.ID] = true
+		}
+	}
+
+	modToIDs := map[string]map[string]*osv.Entry{}
+	for _, vuln := range vulns {
+		if !called[vuln.OSV.ID] {
+			if _, ok := modToIDs[vuln.ModPath]; !ok {
+				modToIDs[vuln.ModPath] = map[string]*osv.Entry{}
+			}
+			// keep only one vuln.OSV instance for the same ID.
+			modToIDs[vuln.ModPath][vuln.OSV.ID] = vuln.OSV
+		}
+	}
+	output := map[string][]*osv.Entry{}
+	for m, vulnSet := range modToIDs {
+		var vulns []*osv.Entry
+		for _, vuln := range vulnSet {
+			vulns = append(vulns, vuln)
+		}
+		sort.Slice(vulns, func(i, j int) bool { return vulns[i].ID < vulns[j].ID })
+		output[m] = vulns
+	}
+	return output
+}
+
+func fixed(v *osv.Entry) string {
+	lf := gvc.LatestFixed(v.Affected)
+	if lf != "" && lf[0] != 'v' {
+		lf = "v" + lf
+	}
+	return lf
+}
+
+func toVulns(ci *gvc.CallInfo, unaffectedMods map[string][]*osv.Entry) ([]Vuln, error) {
 	var vulns []Vuln
 
 	for _, vg := range ci.VulnGroups {
 		v0 := vg[0]
-		lf := gvc.LatestFixed(v0.OSV.Affected)
-		if lf != "" && lf[0] != 'v' {
-			lf = "v" + lf
-		}
 		vuln := Vuln{
 			ID:             v0.OSV.ID,
 			PkgPath:        v0.PkgPath,
 			CurrentVersion: ci.ModuleVersions[v0.ModPath],
-			FixedVersion:   lf,
+			FixedVersion:   fixed(v0.OSV),
 			Details:        v0.OSV.Details,
 
 			Aliases: v0.OSV.Aliases,
@@ -119,5 +177,19 @@ func toVulns(ci *gvc.CallInfo) ([]Vuln, error) {
 		}
 		vulns = append(vulns, vuln)
 	}
+	for m, vg := range unaffectedMods {
+		for _, v0 := range vg {
+			vuln := Vuln{
+				ID:             v0.ID,
+				Details:        v0.Details,
+				Aliases:        v0.Aliases,
+				ModPath:        m,
+				URL:            href(v0),
+				CurrentVersion: "",
+				FixedVersion:   fixed(v0),
+			}
+			vulns = append(vulns, vuln)
+		}
+	}
 	return vulns, nil
 }

gopls/internal/vulncheck/command_test.go

@@ -81,6 +81,15 @@ func TestCmd_Run(t *testing.T) {
 						"golang.org/bmod/bvuln.Vuln (bvuln.go:2)\n",
 				},
 			},
+			{
+				Vuln: Vuln{
+					ID:           "GO-2022-03",
+					Details:      "unaffecting vulnerability",
+					ModPath:      "golang.org/amod",
+					URL:          "https://pkg.go.dev/vuln/GO-2022-03",
+					FixedVersion: "v1.0.4",
+				},
+			},
 		}
 		// sort reports for stability before comparison.
 		for _, rpts := range [][]report{got, want} {
@@ -228,6 +237,21 @@ var testClient1 = &mockClient{
 					EcosystemSpecific: osv.EcosystemSpecific{Symbols: []string{"VulnData.Vuln1", "VulnData.Vuln2"}},
 				}},
 			},
+			{
+				ID:      "GO-2022-03",
+				Details: "unaffecting vulnerability",
+				References: []osv.Reference{
+					{
+						Type: "href",
+						URL:  "pkg.go.dev/vuln/GO-2022-01",
+					},
+				},
+				Affected: []osv.Affected{{
+					Package:           osv.Package{Name: "golang.org/amod/avuln"},
+					Ranges:            osv.Affects{{Type: osv.TypeSemver, Events: []osv.RangeEvent{{Introduced: "1.0.0"}, {Fixed: "1.0.4"}, {Introduced: "1.1.2"}}}},
+					EcosystemSpecific: osv.EcosystemSpecific{Symbols: []string{"nonExisting"}},
+				}},
+			},
 		},
 		"golang.org/bmod": {
 			{

internal/lsp/cache/analysis.go

@@ -54,7 +54,7 @@ func (s *snapshot) Analyze(ctx context.Context, id string, analyzers []*source.A
 	return results, nil
 }
 
-type actionHandleKey string
+type actionHandleKey source.Hash
 
 // An action represents one unit of analysis work: the application of
 // one analysis to one package. Actions form a DAG, both within a
@@ -170,7 +170,7 @@ func (act *actionHandle) analyze(ctx context.Context, snapshot *snapshot) ([]*so
 }
 
 func buildActionKey(a *analysis.Analyzer, ph *packageHandle) actionHandleKey {
-	return actionHandleKey(hashContents([]byte(fmt.Sprintf("%p %s", a, string(ph.key)))))
+	return actionHandleKey(source.Hashf("%p%s", a, ph.key[:]))
 }
 
 func (act *actionHandle) String() string {

internal/lsp/cache/cache.go

@@ -6,7 +6,6 @@ package cache
 
 import (
 	"context"
-	"crypto/sha256"
 	"fmt"
 	"go/ast"
 	"go/token"
@@ -55,7 +54,7 @@ type fileHandle struct {
 	modTime time.Time
 	uri     span.URI
 	bytes   []byte
-	hash    string
+	hash    source.Hash
 	err     error
 
 	// size is the file length as reported by Stat, for the purpose of
@@ -139,7 +138,7 @@ func readFile(ctx context.Context, uri span.URI, fi os.FileInfo) (*fileHandle, e
 		size:    fi.Size(),
 		uri:     uri,
 		bytes:   data,
-		hash:    hashContents(data),
+		hash:    source.HashOf(data),
 	}, nil
 }
 
@@ -168,10 +167,6 @@ func (h *fileHandle) URI() span.URI {
 	return h.uri
 }
 
-func (h *fileHandle) Hash() string {
-	return h.hash
-}
-
 func (h *fileHandle) FileIdentity() source.FileIdentity {
 	return source.FileIdentity{
 		URI:  h.uri,
@@ -183,16 +178,6 @@ func (h *fileHandle) Read() ([]byte, error) {
 	return h.bytes, h.err
 }
 
-// hashContents returns a string of hex digits denoting the hash of contents.
-//
-// TODO(adonovan): opt: use [32]byte array as a value more widely and convert
-// to hex digits on demand (rare). The array is larger when it appears as a
-// struct field (32B vs 16B) but smaller overall (string data is 64B), has
-// better locality, and is more efficiently hashed by runtime maps.
-func hashContents(contents []byte) string {
-	return fmt.Sprintf("%64x", sha256.Sum256(contents))
-}
-
 var cacheIndex, sessionIndex, viewIndex int64
 
 func (c *Cache) ID() string                     { return c.id }