[Extraction] prep for go.mod & go.work FilePath ReplaceDirective work

cmd/fetch_repo/BUILD.bazel

@@ -3,7 +3,9 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
 go_library(
     name = "fetch_repo_lib",
     srcs = [
+        "copy_tree.go",
         "fetch_repo.go",
+        "go_mod_download.go",
         "module.go",
         "vcs.go",
     ],
@@ -33,8 +35,10 @@ filegroup(
     testonly = True,
     srcs = [
         "BUILD.bazel",
+        "copy_tree.go",
         "fetch_repo.go",
         "fetch_repo_test.go",
+        "go_mod_download.go",
         "module.go",
         "vcs.go",
     ],

cmd/fetch_repo/copy_tree.go

@@ -0,0 +1,59 @@
+/* Copyright 2019 The Bazel Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func copyTree(destRoot, srcRoot string) error {
+	return filepath.Walk(srcRoot, func(src string, info os.FileInfo, e error) (err error) {
+		if e != nil {
+			return e
+		}
+		rel, err := filepath.Rel(srcRoot, src)
+		if err != nil {
+			return err
+		}
+		if rel == "." {
+			return nil
+		}
+		dest := filepath.Join(destRoot, rel)
+
+		if info.IsDir() {
+			return os.Mkdir(dest, 0o777)
+		} else {
+			r, err := os.Open(src)
+			if err != nil {
+				return err
+			}
+			defer r.Close()
+			w, err := os.Create(dest)
+			if err != nil {
+				return err
+			}
+			defer func() {
+				if cerr := w.Close(); err == nil && cerr != nil {
+					err = cerr
+				}
+			}()
+			_, err = io.Copy(w, r)
+			return err
+		}
+	})
+}

cmd/fetch_repo/fetch_repo.go

@@ -4,7 +4,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-   http://www.apache.org/licenses/LICENSE-2.0
+  http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

cmd/fetch_repo/go_mod_download.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+type GoModDownloadResult struct {
+	Dir   string
+	Sum   string
+	Error string
+}
+
+func findGoPath() string {
+	// Locate the go binary. If GOROOT is set, we'll use that one; otherwise,
+	// we'll use PATH.
+	goPath := "go"
+	if runtime.GOOS == "windows" {
+		goPath += ".exe"
+	}
+	if goroot, ok := os.LookupEnv("GOROOT"); ok {
+		goPath = filepath.Join(goroot, "bin", goPath)
+	}
+	return goPath
+}
+
+func runGoModDownload(dl *GoModDownloadResult, dest string, importpath string, version string) error {
+	buf := &bytes.Buffer{}
+	bufErr := &bytes.Buffer{}
+	cmd := exec.Command(findGoPath(), "mod", "download", "-json")
+	cmd.Dir = dest
+	cmd.Args = append(cmd.Args, "-modcacherw")
+
+	if version != "" && importpath != "" {
+		cmd.Args = append(cmd.Args, importpath+"@"+version)
+	}
+
+	cmd.Stdout = buf
+	cmd.Stderr = bufErr
+	fmt.Printf("Running: %s %s\n", cmd.Path, strings.Join(cmd.Args, " "))
+	dlErr := cmd.Run()
+	if dlErr != nil {
+		if _, ok := dlErr.(*exec.ExitError); !ok {
+			if bufErr.Len() > 0 {
+				return fmt.Errorf("!%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes())
+			} else {
+				return fmt.Errorf("!!%s %s: %v", cmd.Path, strings.Join(cmd.Args, " "), dlErr)
+			}
+		}
+	}
+
+	// Parse the JSON output.
+	if err := json.Unmarshal(buf.Bytes(), &dl); err != nil {
+		if bufErr.Len() > 0 {
+			return fmt.Errorf("3%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes())
+		} else {
+			return fmt.Errorf("4%s %s: error parsing JSON: %v error: %v", cmd.Path, strings.Join(cmd.Args, " "), buf, err)
+		}
+	}
+	if dl.Error != "" {
+		return errors.New(dl.Error)
+	}
+	if dlErr != nil {
+		return dlErr
+	}
+
+	fmt.Printf("Downloaded: %s\n", dl.Dir)
+
+	return nil
+}

cmd/fetch_repo/module.go

@@ -16,19 +16,9 @@ limitations under the License.
 package main
 
 import (
-	"bytes"
-	"encoding/json"
-	"errors"
 	"fmt"
-	"go/build"
-	"io"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"strings"
-
 	"golang.org/x/mod/sumdb/dirhash"
+	"os"
 )
 
 func fetchModule(dest, importpath, version, sum string) error {
@@ -39,30 +29,11 @@ func fetchModule(dest, importpath, version, sum string) error {
 		return fmt.Errorf("-version must be a complete semantic version. %q is a prefix.", version)
 	}
 
-	// Locate the go binary. If GOROOT is set, we'll use that one; otherwise,
-	// we'll use PATH.
-	goPath := "go"
-	if runtime.GOOS == "windows" {
-		goPath += ".exe"
-	}
-	if goroot, ok := os.LookupEnv("GOROOT"); ok {
-		goPath = filepath.Join(goroot, "bin", goPath)
-	}
-
-	// Check whether -modcacherw is supported.
-	// Assume that fetch_repo was built with the same version of Go we're running.
-	modcacherw := false
-	for _, tag := range build.Default.ReleaseTags {
-		if tag == "go1.14" {
-			modcacherw = true
-			break
-		}
-	}
-
 	// Download the module. In Go 1.11, this command must be run in a module,
 	// so we create a dummy module in the current directory (which should be
 	// empty).
 	w, err := os.OpenFile("go.mod", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)
+
 	if err != nil {
 		return fmt.Errorf("error creating temporary go.mod: %v", err)
 	}
@@ -75,44 +46,12 @@ func fetchModule(dest, importpath, version, sum string) error {
 		return fmt.Errorf("error closing temporary go.mod: %v", err)
 	}
 
-	buf := &bytes.Buffer{}
-	bufErr := &bytes.Buffer{}
-	cmd := exec.Command(goPath, "mod", "download", "-json")
-	if modcacherw {
-		cmd.Args = append(cmd.Args, "-modcacherw")
-	}
-	cmd.Args = append(cmd.Args, importpath+"@"+version)
-	cmd.Stdout = buf
-	cmd.Stderr = bufErr
-	dlErr := cmd.Run()
+	var dl = GoModDownloadResult{}
+	err = runGoModDownload(&dl, dest, importpath, version)
 	os.Remove("go.mod")
-	if dlErr != nil {
-		if _, ok := dlErr.(*exec.ExitError); !ok {
-			if bufErr.Len() > 0 {
-				return fmt.Errorf("%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes())
-			} else {
-				return fmt.Errorf("%s %s: %v", cmd.Path, strings.Join(cmd.Args, " "), dlErr)
-			}
-		}
-	}
 
-	// Parse the JSON output.
-	var dl struct{ Dir, Sum, Error string }
-	if err := json.Unmarshal(buf.Bytes(), &dl); err != nil {
-		if bufErr.Len() > 0 {
-			return fmt.Errorf("%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes())
-		} else {
-			return fmt.Errorf("%s %s: %v", cmd.Path, strings.Join(cmd.Args, " "), err)
-		}
-	}
-	if dl.Error != "" {
-		return errors.New(dl.Error)
-	}
-	if dlErr != nil {
-		return dlErr
-	}
-	if dl.Sum != sum {
-		return fmt.Errorf("downloaded module with sum %s; expected sum %s", dl.Sum, sum)
+	if err != nil {
+		return err
 	}
 
 	// Copy the module to the destination.
@@ -134,43 +73,6 @@ func fetchModule(dest, importpath, version, sum string) error {
 	return nil
 }
 
-func copyTree(destRoot, srcRoot string) error {
-	return filepath.Walk(srcRoot, func(src string, info os.FileInfo, e error) (err error) {
-		if e != nil {
-			return e
-		}
-		rel, err := filepath.Rel(srcRoot, src)
-		if err != nil {
-			return err
-		}
-		if rel == "." {
-			return nil
-		}
-		dest := filepath.Join(destRoot, rel)
-
-		if info.IsDir() {
-			return os.Mkdir(dest, 0o777)
-		} else {
-			r, err := os.Open(src)
-			if err != nil {
-				return err
-			}
-			defer r.Close()
-			w, err := os.Create(dest)
-			if err != nil {
-				return err
-			}
-			defer func() {
-				if cerr := w.Close(); err == nil && cerr != nil {
-					err = cerr
-				}
-			}()
-			_, err = io.Copy(w, r)
-			return err
-		}
-	})
-}
-
 // semantic version parsing functions below this point were copied from
 // cmd/go/internal/semver and cmd/go/internal/modload at go1.12beta2.
 

internal/go_repository_tools_srcs.bzl

@@ -8,7 +8,9 @@ GO_REPOSITORY_TOOLS_SRCS = [
     Label("//cmd/autogazelle:client_unix.go"),
     Label("//cmd/autogazelle:server_unix.go"),
     Label("//cmd/fetch_repo:BUILD.bazel"),
+    Label("//cmd/fetch_repo:copy_tree.go"),
     Label("//cmd/fetch_repo:fetch_repo.go"),
+    Label("//cmd/fetch_repo:go_mod_download.go"),
     Label("//cmd/fetch_repo:module.go"),
     Label("//cmd/fetch_repo:vcs.go"),
     Label("//cmd/gazelle:BUILD.bazel"),