go/internal/gcimporter: update iimport.go to support type parameters

This CL pulls in the latest changes from go/internal/gcimporter, while
avoiding breaking the build on older go versions. To help maintain
compatibility with older Go versions while minimizing the diff with the
standard library importer, the internal/typeparams package was
significantly expanded.

I decided to use type aliases in the internal/typeparams package on Go
version >= go1.18, and placeholder types on Go version < go1.18. This
reduces the amount of copying needed in the APIs, though it might not be
the best decision if we ever decide to export this package.
Documentation was also updated to be more concise and specific to the Go
version being used.

In order to actually fix the x/tools Trybot for packages using generics
in the standard library, we need to switch from the 'typeparams' build
constraint to the 'go1.18' build constraint. This means if we make any
additional API changes in go/types we'll have to submit them with a
broken x/tools Trybot and then immediately fix the x/tools build.

Change-Id: Ifa0b1c37b89dc549ee295fa3a959f03deda86e56
Reviewed-on: https://go-review.googlesource.com/c/tools/+/349949
Trust: Robert Findley <rfindley@google.com>
Run-TryBot: Robert Findley <rfindley@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Robert Griesemer <gri@golang.org>

go/internal/gcimporter/bimport.go

@@ -1029,6 +1029,7 @@ func predeclared() []types.Type {
 			// used internally by gc; never used by this package or in .a files
 			anyType{},
 		}
+		predecl = append(predecl, additionalPredeclared()...)
 	})
 	return predecl
 }

go/internal/gcimporter/iimport.go

@@ -18,6 +18,8 @@ import (
 	"go/types"
 	"io"
 	"sort"
+
+	"golang.org/x/tools/internal/typeparams"
 )
 
 type intReader struct {
@@ -41,6 +43,21 @@ func (r *intReader) uint64() uint64 {
 	return i
 }
 
+// Keep this in sync with constants in iexport.go.
+const (
+	iexportVersionGo1_11 = 0
+	iexportVersionPosCol = 1
+	// TODO: before release, change this back to 2.
+	iexportVersionGenerics = iexportVersionPosCol
+
+	iexportVersionCurrent = iexportVersionGenerics
+)
+
+type ident struct {
+	pkg  string
+	name string
+}
+
 const predeclReserved = 32
 
 type itag uint64
@@ -56,6 +73,9 @@ const (
 	signatureType
 	structType
 	interfaceType
+	typeParamType
+	instType
+	unionType
 )
 
 // IImportData imports a package from the serialized package data
@@ -101,9 +121,13 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data
 
 	version = int64(r.uint64())
 	switch version {
-	case currentVersion, 0:
+	case /* iexportVersionGenerics, */ iexportVersionPosCol, iexportVersionGo1_11:
 	default:
-		errorf("unknown iexport format version %d", version)
+		if version > iexportVersionGenerics {
+			errorf("unstable iexport format version %d, just rebuild compiler and std library", version)
+		} else {
+			errorf("unknown iexport format version %d", version)
+		}
 	}
 
 	sLen := int64(r.uint64())
@@ -115,8 +139,9 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data
 	r.Seek(sLen+dLen, io.SeekCurrent)
 
 	p := iimporter{
-		ipath:   path,
-		version: int(version),
+		exportVersion: version,
+		ipath:         path,
+		version:       int(version),
 
 		stringData:  stringData,
 		stringCache: make(map[uint64]string),
@@ -125,6 +150,9 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data
 		declData: declData,
 		pkgIndex: make(map[*types.Package]map[string]uint64),
 		typCache: make(map[uint64]types.Type),
+		// Separate map for typeparams, keyed by their package and unique
+		// name (name with subscript).
+		tparamIndex: make(map[ident]types.Type),
 
 		fake: fakeFileSet{
 			fset:  fset,
@@ -216,16 +244,18 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data
 }
 
 type iimporter struct {
-	ipath   string
-	version int
+	exportVersion int64
+	ipath         string
+	version       int
 
 	stringData  []byte
 	stringCache map[uint64]string
 	pkgCache    map[uint64]*types.Package
 
-	declData []byte
-	pkgIndex map[*types.Package]map[string]uint64
-	typCache map[uint64]types.Type
+	declData    []byte
+	pkgIndex    map[*types.Package]map[string]uint64
+	typCache    map[uint64]types.Type
+	tparamIndex map[ident]types.Type
 
 	fake          fakeFileSet
 	interfaceList []*types.Interface
@@ -315,17 +345,27 @@ func (r *importReader) obj(name string) {
 
 		r.declare(types.NewConst(pos, r.currPkg, name, typ, val))
 
-	case 'F':
+	case 'F', 'G':
+		var tparams []*typeparams.TypeParam
+		if tag == 'G' {
+			tparams = r.tparamList()
+		}
 		sig := r.signature(nil)
-
+		typeparams.SetForSignature(sig, tparams)
 		r.declare(types.NewFunc(pos, r.currPkg, name, sig))
 
-	case 'T':
+	case 'T', 'U':
 		// Types can be recursive. We need to setup a stub
 		// declaration before recursing.
 		obj := types.NewTypeName(pos, r.currPkg, name, nil)
 		named := types.NewNamed(obj, nil, nil)
+		// Declare obj before calling r.tparamList, so the new type name is recognized
+		// if used in the constraint of one of its own typeparams (see #48280).
 		r.declare(obj)
+		if tag == 'U' {
+			tparams := r.tparamList()
+			typeparams.SetForNamed(named, tparams)
+		}
 
 		underlying := r.p.typAt(r.uint64(), named).Underlying()
 		named.SetUnderlying(underlying)
@@ -337,10 +377,46 @@ func (r *importReader) obj(name string) {
 				recv := r.param()
 				msig := r.signature(recv)
 
+				// If the receiver has any targs, set those as the
+				// rparams of the method (since those are the
+				// typeparams being used in the method sig/body).
+				targs := typeparams.NamedTypeArgs(baseType(msig.Recv().Type()))
+				if len(targs) > 0 {
+					rparams := make([]*typeparams.TypeParam, len(targs))
+					for i := range rparams {
+						rparams[i], _ = targs[i].(*typeparams.TypeParam)
+					}
+					typeparams.SetRecvTypeParams(msig, rparams)
+				}
+
 				named.AddMethod(types.NewFunc(mpos, r.currPkg, mname, msig))
 			}
 		}
 
+	case 'P':
+		// We need to "declare" a typeparam in order to have a name that
+		// can be referenced recursively (if needed) in the type param's
+		// bound.
+		if r.p.exportVersion < iexportVersionGenerics {
+			errorf("unexpected type param type")
+		}
+		name0, sub := parseSubscript(name)
+		tn := types.NewTypeName(pos, r.currPkg, name0, nil)
+		t := typeparams.NewTypeParam(tn, nil)
+		if sub == 0 {
+			errorf("missing subscript")
+		}
+
+		// TODO(rfindley): can we use a different, stable ID?
+		// t.SetId(sub)
+
+		// To handle recursive references to the typeparam within its
+		// bound, save the partial type in tparamIndex before reading the bounds.
+		id := ident{r.currPkg.Name(), name}
+		r.p.tparamIndex[id] = t
+
+		typeparams.SetTypeParamConstraint(t, r.typ())
+
 	case 'V':
 		typ := r.typ()
 
@@ -618,6 +694,49 @@ func (r *importReader) doType(base *types.Named) types.Type {
 		typ := newInterface(methods, embeddeds)
 		r.p.interfaceList = append(r.p.interfaceList, typ)
 		return typ
+
+	case typeParamType:
+		if r.p.exportVersion < iexportVersionGenerics {
+			errorf("unexpected type param type")
+		}
+		pkg, name := r.qualifiedIdent()
+		id := ident{pkg.Name(), name}
+		if t, ok := r.p.tparamIndex[id]; ok {
+			// We're already in the process of importing this typeparam.
+			return t
+		}
+		// Otherwise, import the definition of the typeparam now.
+		r.p.doDecl(pkg, name)
+		return r.p.tparamIndex[id]
+
+	case instType:
+		if r.p.exportVersion < iexportVersionGenerics {
+			errorf("unexpected instantiation type")
+		}
+		// pos does not matter for instances: they are positioned on the original
+		// type.
+		_ = r.pos()
+		len := r.uint64()
+		targs := make([]types.Type, len)
+		for i := range targs {
+			targs[i] = r.typ()
+		}
+		baseType := r.typ()
+		// The imported instantiated type doesn't include any methods, so
+		// we must always use the methods of the base (orig) type.
+		// TODO provide a non-nil *Environment
+		t, _ := typeparams.Instantiate(nil, baseType, targs, false)
+		return t
+
+	case unionType:
+		if r.p.exportVersion < iexportVersionGenerics {
+			errorf("unexpected instantiation type")
+		}
+		terms := make([]*typeparams.Term, r.uint64())
+		for i := range terms {
+			terms[i] = typeparams.NewTerm(r.bool(), r.typ())
+		}
+		return typeparams.NewUnion(terms)
 	}
 }
 
@@ -632,6 +751,20 @@ func (r *importReader) signature(recv *types.Var) *types.Signature {
 	return types.NewSignature(recv, params, results, variadic)
 }
 
+func (r *importReader) tparamList() []*typeparams.TypeParam {
+	n := r.uint64()
+	if n == 0 {
+		return nil
+	}
+	xs := make([]*typeparams.TypeParam, n)
+	for i := range xs {
+		// Note: the standard library importer is tolerant of nil types here,
+		// though would panic in SetTypeParams.
+		xs[i] = r.typ().(*typeparams.TypeParam)
+	}
+	return xs
+}
+
 func (r *importReader) paramList() *types.Tuple {
 	xs := make([]*types.Var, r.uint64())
 	for i := range xs {
@@ -674,3 +807,33 @@ func (r *importReader) byte() byte {
 	}
 	return x
 }
+
+func baseType(typ types.Type) *types.Named {
+	// pointer receivers are never types.Named types
+	if p, _ := typ.(*types.Pointer); p != nil {
+		typ = p.Elem()
+	}
+	// receiver base types are always (possibly generic) types.Named types
+	n, _ := typ.(*types.Named)
+	return n
+}
+
+func parseSubscript(name string) (string, uint64) {
+	// Extract the subscript value from the type param name. We export
+	// and import the subscript value, so that all type params have
+	// unique names.
+	sub := uint64(0)
+	startsub := -1
+	for i, r := range name {
+		if '₀' <= r && r < '₀'+10 {
+			if startsub == -1 {
+				startsub = i
+			}
+			sub = sub*10 + uint64(r-'₀')
+		}
+	}
+	if startsub >= 0 {
+		name = name[:startsub]
+	}
+	return name, sub
+}

go/internal/gcimporter/support_go117.go

@@ -0,0 +1,14 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !go1.18
+// +build !go1.18
+
+package gcimporter
+
+import "go/types"
+
+func additionalPredeclared() []types.Type {
+	return nil
+}

go/internal/gcimporter/support_go118.go

@@ -0,0 +1,18 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.18
+// +build go1.18
+
+package gcimporter
+
+import "go/types"
+
+// additionalPredeclared returns additional predeclared types in go.1.18.
+func additionalPredeclared() []types.Type {
+	return []types.Type{
+		// comparable
+		types.Universe.Lookup("comparable").Type(),
+	}
+}

internal/lsp/source/hover.go

@@ -406,7 +406,7 @@ func moduleAtVersion(path string, i *IdentifierInfo) (string, string, bool) {
 func objectString(obj types.Object, qf types.Qualifier, inferred *types.Signature) string {
 	// If the signature type was inferred, prefer the preferred signature with a
 	// comment showing the generic signature.
-	if sig, _ := obj.Type().(*types.Signature); sig != nil && len(typeparams.ForSignature(sig)) > 0 && inferred != nil {
+	if sig, _ := obj.Type().(*types.Signature); sig != nil && typeparams.ForSignature(sig).Len() > 0 && inferred != nil {
 		obj2 := types.NewFunc(obj.Pos(), obj.Pkg(), obj.Name(), inferred)
 		str := types.ObjectString(obj2, qf)
 		// Try to avoid overly long lines.

internal/lsp/source/identifier.go

@@ -377,7 +377,7 @@ func inferredSignature(info *types.Info, path []ast.Node) *types.Signature {
 			// If the IndexExpr is fully instantiated, we consider that 'inference' for
 			// gopls' purposes.
 			sig, _ := info.TypeOf(e).(*types.Signature)
-			if sig != nil && len(typeparams.ForSignature(sig)) == 0 {
+			if sig != nil && typeparams.ForSignature(sig).Len() == 0 {
 				return sig
 			}
 			_, sig = typeparams.GetInferred(info, e)

internal/lsp/testdata/summary_generics.txt.golden → internal/lsp/testdata/summary_go1.18.txt.golden

@@ -20,7 +20,7 @@ DefinitionsCount = 99
 TypeDefinitionsCount = 18
 HighlightsCount = 69
 ReferencesCount = 27
-RenamesCount = 37
+RenamesCount = 41
 PrepareRenamesCount = 7
 SymbolsCount = 5
 WorkspaceSymbolsCount = 20

internal/lsp/tests/tests.go

@@ -47,7 +47,7 @@ var summaryFile = "summary.txt"
 
 func init() {
 	if typeparams.Enabled {
-		summaryFile = "summary_generics.txt"
+		summaryFile = "summary_go1.18.txt"
 	}
 }
 

internal/typeparams/enabled_go117.go

@@ -0,0 +1,12 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !go1.18
+// +build !go1.18
+
+package typeparams
+
+// Enabled reports whether type parameters are enabled in the current build
+// environment.
+const Enabled = false

internal/typeparams/enabled_go118.go

@@ -0,0 +1,15 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.18
+// +build go1.18
+
+package typeparams
+
+// Note: this constant is in a separate file as this is the only acceptable
+// diff between the <1.18 API of this package and the 1.18 API.
+
+// Enabled reports whether type parameters are enabled in the current build
+// environment.
+const Enabled = true