From d4627bea012def6c1eda50b577f9dcd8c81471b8 Mon Sep 17 00:00:00 2001 From: Stefan Penner Date: Mon, 8 Apr 2024 16:46:34 -0600 Subject: [PATCH] [Extraction] prep for go.mod & go.work FilePath ReplaceDirective work --- cmd/fetch_repo/BUILD.bazel | 4 + cmd/fetch_repo/copy_tree.go | 59 ++++++++++++++ cmd/fetch_repo/fetch_repo.go | 2 +- cmd/fetch_repo/go_mod_download.go | 77 ++++++++++++++++++ cmd/fetch_repo/module.go | 110 ++------------------------ internal/go_repository_tools_srcs.bzl | 2 + 6 files changed, 149 insertions(+), 105 deletions(-) create mode 100644 cmd/fetch_repo/copy_tree.go create mode 100644 cmd/fetch_repo/go_mod_download.go diff --git a/cmd/fetch_repo/BUILD.bazel b/cmd/fetch_repo/BUILD.bazel index c488904c4..f0ffdb009 100644 --- a/cmd/fetch_repo/BUILD.bazel +++ b/cmd/fetch_repo/BUILD.bazel @@ -3,7 +3,9 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") go_library( name = "fetch_repo_lib", srcs = [ + "copy_tree.go", "fetch_repo.go", + "go_mod_download.go", "module.go", "vcs.go", ], @@ -33,8 +35,10 @@ filegroup( testonly = True, srcs = [ "BUILD.bazel", + "copy_tree.go", "fetch_repo.go", "fetch_repo_test.go", + "go_mod_download.go", "module.go", "vcs.go", ], diff --git a/cmd/fetch_repo/copy_tree.go b/cmd/fetch_repo/copy_tree.go new file mode 100644 index 000000000..1d2bf4d9c --- /dev/null +++ b/cmd/fetch_repo/copy_tree.go @@ -0,0 +1,59 @@ +/* Copyright 2019 The Bazel Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "io" + "os" + "path/filepath" +) + +func copyTree(destRoot, srcRoot string) error { + return filepath.Walk(srcRoot, func(src string, info os.FileInfo, e error) (err error) { + if e != nil { + return e + } + rel, err := filepath.Rel(srcRoot, src) + if err != nil { + return err + } + if rel == "." { + return nil + } + dest := filepath.Join(destRoot, rel) + + if info.IsDir() { + return os.Mkdir(dest, 0o777) + } else { + r, err := os.Open(src) + if err != nil { + return err + } + defer r.Close() + w, err := os.Create(dest) + if err != nil { + return err + } + defer func() { + if cerr := w.Close(); err == nil && cerr != nil { + err = cerr + } + }() + _, err = io.Copy(w, r) + return err + } + }) +} diff --git a/cmd/fetch_repo/fetch_repo.go b/cmd/fetch_repo/fetch_repo.go index e45a9260a..f494bf81a 100644 --- a/cmd/fetch_repo/fetch_repo.go +++ b/cmd/fetch_repo/fetch_repo.go @@ -4,7 +4,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/cmd/fetch_repo/go_mod_download.go b/cmd/fetch_repo/go_mod_download.go new file mode 100644 index 000000000..6d9c968c3 --- /dev/null +++ b/cmd/fetch_repo/go_mod_download.go @@ -0,0 +1,77 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +type GoModDownloadResult struct { + Dir string + Sum string + Error string +} + +func findGoPath() string { + // Locate the go binary. If GOROOT is set, we'll use that one; otherwise, + // we'll use PATH. + goPath := "go" + if runtime.GOOS == "windows" { + goPath += ".exe" + } + if goroot, ok := os.LookupEnv("GOROOT"); ok { + goPath = filepath.Join(goroot, "bin", goPath) + } + return goPath +} + +func runGoModDownload(dl *GoModDownloadResult, dest string, importpath string, version string) error { + buf := &bytes.Buffer{} + bufErr := &bytes.Buffer{} + cmd := exec.Command(findGoPath(), "mod", "download", "-json") + cmd.Dir = dest + cmd.Args = append(cmd.Args, "-modcacherw") + + if version != "" && importpath != "" { + cmd.Args = append(cmd.Args, importpath+"@"+version) + } + + cmd.Stdout = buf + cmd.Stderr = bufErr + fmt.Printf("Running: %s %s\n", cmd.Path, strings.Join(cmd.Args, " ")) + dlErr := cmd.Run() + if dlErr != nil { + if _, ok := dlErr.(*exec.ExitError); !ok { + if bufErr.Len() > 0 { + return fmt.Errorf("!%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes()) + } else { + return fmt.Errorf("!!%s %s: %v", cmd.Path, strings.Join(cmd.Args, " "), dlErr) + } + } + } + + // Parse the JSON output. + if err := json.Unmarshal(buf.Bytes(), &dl); err != nil { + if bufErr.Len() > 0 { + return fmt.Errorf("3%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes()) + } else { + return fmt.Errorf("4%s %s: error parsing JSON: %v error: %v", cmd.Path, strings.Join(cmd.Args, " "), buf, err) + } + } + if dl.Error != "" { + return errors.New(dl.Error) + } + if dlErr != nil { + return dlErr + } + + fmt.Printf("Downloaded: %s\n", dl.Dir) + + return nil +} diff --git a/cmd/fetch_repo/module.go b/cmd/fetch_repo/module.go index 3ff2fedc9..5442bb2fa 100644 --- a/cmd/fetch_repo/module.go +++ b/cmd/fetch_repo/module.go @@ -16,19 +16,9 @@ limitations under the License. package main import ( - "bytes" - "encoding/json" - "errors" "fmt" - "go/build" - "io" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "golang.org/x/mod/sumdb/dirhash" + "os" ) func fetchModule(dest, importpath, version, sum string) error { @@ -39,30 +29,11 @@ func fetchModule(dest, importpath, version, sum string) error { return fmt.Errorf("-version must be a complete semantic version. %q is a prefix.", version) } - // Locate the go binary. If GOROOT is set, we'll use that one; otherwise, - // we'll use PATH. - goPath := "go" - if runtime.GOOS == "windows" { - goPath += ".exe" - } - if goroot, ok := os.LookupEnv("GOROOT"); ok { - goPath = filepath.Join(goroot, "bin", goPath) - } - - // Check whether -modcacherw is supported. - // Assume that fetch_repo was built with the same version of Go we're running. - modcacherw := false - for _, tag := range build.Default.ReleaseTags { - if tag == "go1.14" { - modcacherw = true - break - } - } - // Download the module. In Go 1.11, this command must be run in a module, // so we create a dummy module in the current directory (which should be // empty). w, err := os.OpenFile("go.mod", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666) + if err != nil { return fmt.Errorf("error creating temporary go.mod: %v", err) } @@ -75,44 +46,12 @@ func fetchModule(dest, importpath, version, sum string) error { return fmt.Errorf("error closing temporary go.mod: %v", err) } - buf := &bytes.Buffer{} - bufErr := &bytes.Buffer{} - cmd := exec.Command(goPath, "mod", "download", "-json") - if modcacherw { - cmd.Args = append(cmd.Args, "-modcacherw") - } - cmd.Args = append(cmd.Args, importpath+"@"+version) - cmd.Stdout = buf - cmd.Stderr = bufErr - dlErr := cmd.Run() + var dl = GoModDownloadResult{} + err = runGoModDownload(&dl, dest, importpath, version) os.Remove("go.mod") - if dlErr != nil { - if _, ok := dlErr.(*exec.ExitError); !ok { - if bufErr.Len() > 0 { - return fmt.Errorf("%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes()) - } else { - return fmt.Errorf("%s %s: %v", cmd.Path, strings.Join(cmd.Args, " "), dlErr) - } - } - } - // Parse the JSON output. - var dl struct{ Dir, Sum, Error string } - if err := json.Unmarshal(buf.Bytes(), &dl); err != nil { - if bufErr.Len() > 0 { - return fmt.Errorf("%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes()) - } else { - return fmt.Errorf("%s %s: %v", cmd.Path, strings.Join(cmd.Args, " "), err) - } - } - if dl.Error != "" { - return errors.New(dl.Error) - } - if dlErr != nil { - return dlErr - } - if dl.Sum != sum { - return fmt.Errorf("downloaded module with sum %s; expected sum %s", dl.Sum, sum) + if err != nil { + return err } // Copy the module to the destination. @@ -134,43 +73,6 @@ func fetchModule(dest, importpath, version, sum string) error { return nil } -func copyTree(destRoot, srcRoot string) error { - return filepath.Walk(srcRoot, func(src string, info os.FileInfo, e error) (err error) { - if e != nil { - return e - } - rel, err := filepath.Rel(srcRoot, src) - if err != nil { - return err - } - if rel == "." { - return nil - } - dest := filepath.Join(destRoot, rel) - - if info.IsDir() { - return os.Mkdir(dest, 0o777) - } else { - r, err := os.Open(src) - if err != nil { - return err - } - defer r.Close() - w, err := os.Create(dest) - if err != nil { - return err - } - defer func() { - if cerr := w.Close(); err == nil && cerr != nil { - err = cerr - } - }() - _, err = io.Copy(w, r) - return err - } - }) -} - // semantic version parsing functions below this point were copied from // cmd/go/internal/semver and cmd/go/internal/modload at go1.12beta2. diff --git a/internal/go_repository_tools_srcs.bzl b/internal/go_repository_tools_srcs.bzl index b135dd24c..218f380c4 100644 --- a/internal/go_repository_tools_srcs.bzl +++ b/internal/go_repository_tools_srcs.bzl @@ -8,7 +8,9 @@ GO_REPOSITORY_TOOLS_SRCS = [ Label("//cmd/autogazelle:client_unix.go"), Label("//cmd/autogazelle:server_unix.go"), Label("//cmd/fetch_repo:BUILD.bazel"), + Label("//cmd/fetch_repo:copy_tree.go"), Label("//cmd/fetch_repo:fetch_repo.go"), + Label("//cmd/fetch_repo:go_mod_download.go"), Label("//cmd/fetch_repo:module.go"), Label("//cmd/fetch_repo:vcs.go"), Label("//cmd/gazelle:BUILD.bazel"),