cmd/go/internal/modfetch: vgo.fetch becomes modfetch.Download

Refactoring so that all the cache directory management is in package modfetch.

While here, rename go.modverify to go.sum,
in preparation for forcing go.sum checking on
all the time.

Vgo will automatically move content from
go.modverify into go.sum when it sees it.

For golang/go#25525.

Change-Id: Id03286d52ca4923b35ac14b639e5885571397bba
Reviewed-on: https://go-review.googlesource.com/121298
Reviewed-by: Bryan C. Mills <bcmills@google.com>

vendor/cmd/go/internal/modfetch/cache.go

@@ -21,7 +21,7 @@ import (
 
 var QuietLookup bool // do not print about lookups
 
-var CacheRoot string // $GOPATH/src/mod/cache
+var SrcMod string // $GOPATH/src/mod; set by package vgo
 
 // A cachingRepo is a cache around an underlying Repo,
 // avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not Zip).
@@ -230,7 +230,7 @@ func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error
 		return "", nil, errNotCached
 	}
 	rev = rev[:12]
-	dir, err := os.Open(filepath.Join(CacheRoot, path, "@v"))
+	dir, err := os.Open(filepath.Join(SrcMod, "cache/download", path, "@v"))
 	if err != nil {
 		return "", nil, errNotCached
 	}
@@ -279,10 +279,10 @@ func readDiskGoMod(path, rev string) (file string, data []byte, err error) {
 // If the read fails, the caller can use
 // writeDiskCache(file, data) to write a new cache entry.
 func readDiskCache(path, rev, suffix string) (file string, data []byte, err error) {
-	if !semver.IsValid(rev) || CacheRoot == "" {
+	if !semver.IsValid(rev) || SrcMod == "" {
 		return "", nil, errNotCached
 	}
-	file = filepath.Join(CacheRoot, path, "@v", rev+"."+suffix)
+	file = filepath.Join(SrcMod, "cache/download", path, "@v", rev+"."+suffix)
 	data, err = ioutil.ReadFile(file)
 	if err != nil {
 		return file, nil, errNotCached

vendor/cmd/go/internal/vgo/fetch.go → vendor/cmd/go/internal/modfetch/fetch.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package vgo
+package modfetch
 
 import (
 	"archive/zip"
@@ -17,26 +17,13 @@ import (
 
 	"cmd/go/internal/base"
 	"cmd/go/internal/dirhash"
-	"cmd/go/internal/modfetch"
 	"cmd/go/internal/module"
-	"cmd/go/internal/semver"
 )
 
-// fetch returns the directory in the local download cache
-// holding the root of mod's source tree.
-// It downloads the module if needed.
-func fetch(mod module.Version) (dir string, err error) {
-	if r := Replacement(mod); r.Path != "" {
-		if r.Version == "" {
-			dir = r.Path
-			if !filepath.IsAbs(dir) {
-				dir = filepath.Join(ModRoot, dir)
-			}
-			return dir, nil
-		}
-		mod = r
-	}
-
+// Download downloads the specific module version to the
+// local download cache and returns the name of the directory
+// corresponding to the root of the module's file tree.
+func Download(mod module.Version) (dir string, err error) {
 	modpath := mod.Path + "@" + mod.Version
 	dir = filepath.Join(SrcMod, modpath)
 	if files, _ := ioutil.ReadDir(dir); len(files) == 0 {
@@ -55,17 +42,17 @@ func fetch(mod module.Version) (dir string, err error) {
 				return "", err
 			}
 		}
-		if err := modfetch.Unzip(dir, zipfile, modpath, 0); err != nil {
+		if err := Unzip(dir, zipfile, modpath, 0); err != nil {
 			fmt.Fprintf(os.Stderr, "-> %s\n", err)
 			return "", err
 		}
 	}
-	checkModHash(mod)
+	checkSum(mod)
 	return dir, nil
 }
 
 func downloadZip(mod module.Version, target string) error {
-	repo, err := modfetch.Lookup(mod.Path)
+	repo, err := Lookup(mod.Path)
 	if err != nil {
 		return err
 	}
@@ -113,23 +100,38 @@ func downloadZip(mod module.Version, target string) error {
 	return ioutil.WriteFile(target+"hash", []byte(hash), 0666)
 }
 
-var useModHash = false
-var modHash map[module.Version][]string
+var (
+	GoSumFile string                      // path to go.sum; set by package vgo
+	modverify string                      // path to go.modverify, to be deleted
+	goSum     map[module.Version][]string // content of go.sum file (+ go.modverify if present)
+	useGoSum  bool                        // whether to use go.sum at all
+)
 
-func initModHash() {
-	if modHash != nil {
+func initGoSum() {
+	if goSum != nil || GoSumFile == "" {
 		return
 	}
-	modHash = make(map[module.Version][]string)
-	file := filepath.Join(ModRoot, "go.modverify")
-	data, err := ioutil.ReadFile(file)
-	if err != nil && os.IsNotExist(err) {
-		return
+	goSum = make(map[module.Version][]string)
+	data, err := ioutil.ReadFile(GoSumFile)
+	if err != nil && !os.IsNotExist(err) {
+		base.Fatalf("vgo: %v", err)
 	}
 	if err != nil {
-		base.Fatalf("vgo: %v", err)
+		return
 	}
-	useModHash = true
+	useGoSum = true
+	readGoSum(GoSumFile, data)
+
+	// Add old go.modverify file.
+	// We'll delete go.modverify in WriteGoSum.
+	alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify"
+	if data, err := ioutil.ReadFile(alt); err == nil {
+		readGoSum(alt, data)
+		modverify = alt
+	}
+}
+
+func readGoSum(file string, data []byte) {
 	lineno := 0
 	for len(data) > 0 {
 		var line []byte
@@ -146,16 +148,16 @@ func initModHash() {
 			continue
 		}
 		if len(f) != 3 {
-			base.Fatalf("vgo: malformed go.modverify:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
+			base.Fatalf("vgo: malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
 		}
 		mod := module.Version{Path: f[0], Version: f[1]}
-		modHash[mod] = append(modHash[mod], f[2])
+		goSum[mod] = append(goSum[mod], f[2])
 	}
 }
 
-func checkModHash(mod module.Version) {
-	initModHash()
-	if !useModHash {
+func checkSum(mod module.Version) {
+	initGoSum()
+	if !useGoSum {
 		return
 	}
 
@@ -168,65 +170,58 @@ func checkModHash(mod module.Version) {
 		base.Fatalf("vgo: verifying %s %s: unexpected ziphash: %q", mod.Path, mod.Version, h)
 	}
 
-	for _, vh := range modHash[mod] {
+	for _, vh := range goSum[mod] {
 		if h == vh {
 			return
 		}
 		if strings.HasPrefix(vh, "h1:") {
-			base.Fatalf("vgo: verifying %s %s: module hash mismatch\n\tdownloaded:   %v\n\tgo.modverify: %v", mod.Path, mod.Version, h, vh)
+			base.Fatalf("vgo: verifying %s %s: module hash mismatch\n\tdownloaded: %v\n\tgo.sum:     %v", mod.Path, mod.Version, h, vh)
 		}
 	}
-	if len(modHash[mod]) > 0 {
-		fmt.Fprintf(os.Stderr, "warning: verifying %s %s: unknown hashes in go.modverify: %v; adding %v", mod.Path, mod.Version, strings.Join(modHash[mod], ", "), h)
+	if len(goSum[mod]) > 0 {
+		fmt.Fprintf(os.Stderr, "warning: verifying %s %s: unknown hashes in go.sum: %v; adding %v", mod.Path, mod.Version, strings.Join(goSum[mod], ", "), h)
 	}
-	modHash[mod] = append(modHash[mod], h)
+	goSum[mod] = append(goSum[mod], h)
 }
 
-func findModHash(mod module.Version) string {
+// Sum returns the checksum for the downloaded copy of the given module,
+// if present in the download cache.
+func Sum(mod module.Version) string {
 	data, err := ioutil.ReadFile(filepath.Join(SrcMod, "cache/download", mod.Path, "@v", mod.Version+".ziphash"))
 	if err != nil {
 		return ""
 	}
 	return strings.TrimSpace(string(data))
 }
 
-func writeModHash() {
-	if !useModHash {
+// WriteGoSum writes the go.sum file if it needs to be updated.
+func WriteGoSum() {
+	if !useGoSum {
 		return
 	}
 
 	var mods []module.Version
-	for m := range modHash {
+	for m := range goSum {
 		mods = append(mods, m)
 	}
-	sortModules(mods)
+	module.Sort(mods)
 	var buf bytes.Buffer
 	for _, m := range mods {
-		list := modHash[m]
+		list := goSum[m]
 		sort.Strings(list)
 		for _, h := range list {
 			fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h)
 		}
 	}
 
-	file := filepath.Join(ModRoot, "go.modverify")
-	data, _ := ioutil.ReadFile(filepath.Join(ModRoot, "go.modverify"))
-	if bytes.Equal(data, buf.Bytes()) {
-		return
+	data, _ := ioutil.ReadFile(GoSumFile)
+	if !bytes.Equal(data, buf.Bytes()) {
+		if err := ioutil.WriteFile(GoSumFile, buf.Bytes(), 0666); err != nil {
+			base.Fatalf("vgo: writing go.sum: %v", err)
+		}
 	}
 
-	if err := ioutil.WriteFile(file, buf.Bytes(), 0666); err != nil {
-		base.Fatalf("vgo: writing go.modverify: %v", err)
+	if modverify != "" {
+		os.Remove(modverify)
 	}
 }
-
-func sortModules(mods []module.Version) {
-	sort.Slice(mods, func(i, j int) bool {
-		mi := mods[i]
-		mj := mods[j]
-		if mi.Path != mj.Path {
-			return mi.Path < mj.Path
-		}
-		return semver.Compare(mi.Version, mj.Version) < 0
-	})
-}

vendor/cmd/go/internal/module/module.go

@@ -8,6 +8,7 @@ package module
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 	"unicode"
 	"unicode/utf8"
@@ -200,3 +201,15 @@ func MatchPathMajor(v, pathMajor string) bool {
 	}
 	return (pathMajor[0] == '/' || pathMajor[0] == '.') && m == pathMajor[1:]
 }
+
+// Sort sorts the list by Path, breaking ties by comparing Versions.
+func Sort(list []Version) {
+	sort.Slice(list, func(i, j int) bool {
+		mi := list[i]
+		mj := list[j]
+		if mi.Path != mj.Path {
+			return mi.Path < mj.Path
+		}
+		return semver.Compare(mi.Version, mj.Version) < 0
+	})
+}

vendor/cmd/go/internal/vgo/build.go

@@ -161,15 +161,15 @@ func PackageBuildInfo(path string, deps []string) string {
 	for mod := range mdeps {
 		mods = append(mods, mod)
 	}
-	sortModules(mods)
+	module.Sort(mods)
 
 	var buf bytes.Buffer
 	fmt.Fprintf(&buf, "path\t%s\n", path)
 	tv := target.Version
 	if tv == "" {
 		tv = "(devel)"
 	}
-	fmt.Fprintf(&buf, "mod\t%s\t%s\t%s\n", target.Path, tv, findModHash(target))
+	fmt.Fprintf(&buf, "mod\t%s\t%s\t%s\n", target.Path, tv, modfetch.Sum(target))
 	for _, mod := range mods {
 		mv := mod.Version
 		if mv == "" {
@@ -178,11 +178,11 @@ func PackageBuildInfo(path string, deps []string) string {
 		r := Replacement(mod)
 		h := ""
 		if r.Path == "" {
-			h = "\t" + findModHash(mod)
+			h = "\t" + modfetch.Sum(mod)
 		}
 		fmt.Fprintf(&buf, "dep\t%s\t%s%s\n", mod.Path, mod.Version, h)
 		if r.Path != "" {
-			fmt.Fprintf(&buf, "=>\t%s\t%s\t%s\n", r.Path, r.Version, findModHash(r))
+			fmt.Fprintf(&buf, "=>\t%s\t%s\t%s\n", r.Path, r.Version, modfetch.Sum(r))
 		}
 	}
 	return buf.String()

vendor/cmd/go/internal/vgo/init.go

@@ -188,7 +188,7 @@ func InitMod() {
 		os.Rename(srcV, SrcMod)
 	}
 
-	modfetch.CacheRoot = filepath.Join(SrcMod, "cache/download")
+	modfetch.SrcMod = SrcMod
 	codehost.WorkRoot = filepath.Join(SrcMod, "cache/vcs")
 
 	if CmdModInit {
@@ -419,7 +419,7 @@ func findImportComment(file string) string {
 
 // WriteGoMod writes the current build list back to go.mod.
 func WriteGoMod() {
-	writeModHash()
+	modfetch.WriteGoSum()
 
 	if buildList != nil {
 		min, err := mvs.Req(Target, buildList, newReqs())

vendor/cmd/go/internal/vgo/load.go

@@ -627,3 +627,18 @@ func scanDir(path string, tags map[string]bool) (imports_, testImports []string,
 
 	return filter(imports_), filter(testImports), err
 }
+
+func fetch(mod module.Version) (dir string, err error) {
+	if r := Replacement(mod); r.Path != "" {
+		if r.Version == "" {
+			dir = r.Path
+			if !filepath.IsAbs(dir) {
+				dir = filepath.Join(ModRoot, dir)
+			}
+			return dir, nil
+		}
+		mod = r
+	}
+
+	return modfetch.Download(mod)
+}

vendor/cmd/go/vgo_test.go

@@ -671,7 +671,7 @@ func TestVerifyNotDownloaded(t *testing.T) {
 		module x
 		require github.com/pkg/errors v0.8.0
 	`), 0666))
-	tg.must(ioutil.WriteFile(tg.path("x/go.modverify"), []byte(`github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+	tg.must(ioutil.WriteFile(tg.path("x/go.sum"), []byte(`github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 `), 0666))
 	tg.must(ioutil.WriteFile(tg.path("x/x.go"), []byte(`package x`), 0666))
 	tg.cd(tg.path("x"))