Skip to content

Commit

Permalink
POC - go.mod replace with ModulePath
Browse files Browse the repository at this point in the history
  • Loading branch information
Stefan Penner committed Mar 16, 2024
1 parent a2252f8 commit cb5dd43
Show file tree
Hide file tree
Showing 13 changed files with 339 additions and 126 deletions.
4 changes: 4 additions & 0 deletions cmd/fetch_repo/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
go_library(
name = "fetch_repo_lib",
srcs = [
"copy_tree.go",
"fetch_repo.go",
"module.go",
"path.go",
"vcs.go",
],
importpath = "github.com/bazelbuild/bazel-gazelle/cmd/fetch_repo",
Expand All @@ -30,9 +32,11 @@ filegroup(
testonly = True,
srcs = [
"BUILD.bazel",
"copy_tree.go",
"fetch_repo.go",
"fetch_repo_test.go",
"module.go",
"path.go",
"vcs.go",
],
visibility = ["//visibility:public"],
Expand Down
59 changes: 59 additions & 0 deletions cmd/fetch_repo/copy_tree.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/* Copyright 2019 The Bazel Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"io"
"os"
"path/filepath"
)

func copyTree(destRoot, srcRoot string) error {
return filepath.Walk(srcRoot, func(src string, info os.FileInfo, e error) (err error) {
if e != nil {
return e
}
rel, err := filepath.Rel(srcRoot, src)
if err != nil {
return err
}
if rel == "." {
return nil
}
dest := filepath.Join(destRoot, rel)

if info.IsDir() {
return os.Mkdir(dest, 0o777)
} else {
r, err := os.Open(src)
if err != nil {
return err
}
defer r.Close()
w, err := os.Create(dest)
if err != nil {
return err
}
defer func() {
if cerr := w.Close(); err == nil && cerr != nil {
err = cerr
}
}()
_, err = io.Copy(w, r)
return err
}
})
}
32 changes: 29 additions & 3 deletions cmd/fetch_repo/fetch_repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
Expand Down Expand Up @@ -34,6 +34,7 @@ import (
var (
// Common flags
importpath = flag.String("importpath", "", "Go importpath to the repository fetch")
path = flag.String("path", "", "path to a local go module")
dest = flag.String("dest", "", "destination directory")

// Repository flags
Expand All @@ -54,17 +55,42 @@ func main() {
log.SetPrefix("fetch_repo: ")

flag.Parse()
if *importpath == "" {

if *importpath == "" && *path == "" {
log.Fatal("-importpath must be set")
}

if *dest == "" {
log.Fatal("-dest must be set")
}
if flag.NArg() != 0 {
log.Fatal("fetch_repo does not accept positional arguments")
}

if *version != "" {
if *path != "" {
if *importpath != "" {
log.Fatal("-importpath must not be set")
}
if *remote != "" {
log.Fatal("-remote must not be set in module path mode")
}
if *cmd != "" {
log.Fatal("-vcs must not be set in module path mode")
}
if *rev != "" {
log.Fatal("-rev must not be set in module path mode")
}
if *version != "" {
log.Fatal("-version must not be set in module path mode")
}
if *sum != "" {
log.Fatal("-sum must not be set in module path mode")
}

if err := moduleFromPath(*path, *dest); err != nil {
log.Fatal(err)
}
} else if *version != "" {
if *remote != "" {
log.Fatal("-remote must not be set in module mode")
}
Expand Down
11 changes: 11 additions & 0 deletions cmd/fetch_repo/fetch_repo_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,3 +107,14 @@ func TestGetRepoRoot_error(t *testing.T) {
}
}
}

func TestModuleFromPath(t *testing.T) {
// Replace copyTree to avoid copying files.
// copyTree = func(_, _ string) error { return nil }
// Replace execCommand to avoid running go mod download.
// execCommand = func(_ string, _ ...string) error { return nil }

if err := moduleFromPath("from", "dest"); err != nil {
t.Errorf("moduleFromPath: %v", err)
}
}
93 changes: 93 additions & 0 deletions cmd/fetch_repo/go_mod_download.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package main

import (
"bytes"
"encoding/json"
"errors"
"fmt"
"go/build"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
)

type GoModDownloadResult struct {
Dir string
Sum string
Error string
}

func isModeCacherwSupported() bool {
// Check whether -modcacherw is supported.
// Assume that fetch_repo was built with the same version of Go we're running.
modcacherw := false
for _, tag := range build.Default.ReleaseTags {
if tag == "go1.14" {
modcacherw = true
break
}
}

return modcacherw
}
func findGoPath() string {
// Locate the go binary. If GOROOT is set, we'll use that one; otherwise,
// we'll use PATH.
goPath := "go"
if runtime.GOOS == "windows" {
goPath += ".exe"
}
if goroot, ok := os.LookupEnv("GOROOT"); ok {
goPath = filepath.Join(goroot, "bin", goPath)
}
return goPath
}

func runGoModDownload(dl *GoModDownloadResult, dest string, importpath string, version string) error {
buf := &bytes.Buffer{}
bufErr := &bytes.Buffer{}
cmd := exec.Command(findGoPath(), "mod", "download", "-json")
cmd.Dir = dest
if isModeCacherwSupported() {
cmd.Args = append(cmd.Args, "-modcacherw")
}

if version != "" && importpath != "" {
cmd.Args = append(cmd.Args, importpath+"@"+version)
}

cmd.Stdout = buf
cmd.Stderr = bufErr
fmt.Printf("Running: %s %s\n", cmd.Path, strings.Join(cmd.Args, " "))
dlErr := cmd.Run()
if dlErr != nil {
if _, ok := dlErr.(*exec.ExitError); !ok {
if bufErr.Len() > 0 {
return fmt.Errorf("%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes())
} else {
return fmt.Errorf("%s %s: %v", cmd.Path, strings.Join(cmd.Args, " "), dlErr)
}
}
}

// Parse the JSON output.
if err := json.Unmarshal(buf.Bytes(), &dl); err != nil {
if bufErr.Len() > 0 {
return fmt.Errorf("%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes())
} else {
return fmt.Errorf("%s %s: %v", cmd.Path, strings.Join(cmd.Args, " "), err)
}
}
if dl.Error != "" {
return errors.New(dl.Error)
}
if dlErr != nil {
return dlErr
}

fmt.Printf("Downloaded: %s\n", dl.Dir)

return nil
}
110 changes: 7 additions & 103 deletions cmd/fetch_repo/module.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,8 @@ limitations under the License.
package main

import (
"bytes"
"encoding/json"
"errors"
"fmt"
"go/build"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
)

func fetchModule(dest, importpath, version, sum string) error {
Expand All @@ -37,30 +28,11 @@ func fetchModule(dest, importpath, version, sum string) error {
return fmt.Errorf("-version must be a complete semantic version. %q is a prefix.", version)
}

// Locate the go binary. If GOROOT is set, we'll use that one; otherwise,
// we'll use PATH.
goPath := "go"
if runtime.GOOS == "windows" {
goPath += ".exe"
}
if goroot, ok := os.LookupEnv("GOROOT"); ok {
goPath = filepath.Join(goroot, "bin", goPath)
}

// Check whether -modcacherw is supported.
// Assume that fetch_repo was built with the same version of Go we're running.
modcacherw := false
for _, tag := range build.Default.ReleaseTags {
if tag == "go1.14" {
modcacherw = true
break
}
}

// Download the module. In Go 1.11, this command must be run in a module,
// so we create a dummy module in the current directory (which should be
// empty).
w, err := os.OpenFile("go.mod", os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)

if err != nil {
return fmt.Errorf("error creating temporary go.mod: %v", err)
}
Expand All @@ -73,87 +45,19 @@ func fetchModule(dest, importpath, version, sum string) error {
return fmt.Errorf("error closing temporary go.mod: %v", err)
}

buf := &bytes.Buffer{}
bufErr := &bytes.Buffer{}
cmd := exec.Command(goPath, "mod", "download", "-json")
if modcacherw {
cmd.Args = append(cmd.Args, "-modcacherw")
}
cmd.Args = append(cmd.Args, importpath+"@"+version)
cmd.Stdout = buf
cmd.Stderr = bufErr
dlErr := cmd.Run()
os.Remove("go.mod")
if dlErr != nil {
if _, ok := dlErr.(*exec.ExitError); !ok {
if bufErr.Len() > 0 {
return fmt.Errorf("%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes())
} else {
return fmt.Errorf("%s %s: %v", cmd.Path, strings.Join(cmd.Args, " "), dlErr)
}
}
}
defer os.Remove("go.mod")

// Parse the JSON output.
var dl struct{ Dir, Sum, Error string }
if err := json.Unmarshal(buf.Bytes(), &dl); err != nil {
if bufErr.Len() > 0 {
return fmt.Errorf("%s %s: %s", cmd.Path, strings.Join(cmd.Args, " "), bufErr.Bytes())
} else {
return fmt.Errorf("%s %s: %v", cmd.Path, strings.Join(cmd.Args, " "), err)
}
}
if dl.Error != "" {
return errors.New(dl.Error)
}
if dlErr != nil {
return dlErr
}
if dl.Sum != sum {
return fmt.Errorf("downloaded module with sum %s; expected sum %s", dl.Sum, sum)
var dl = GoModDownloadResult{}
err = runGoModDownload(&dl, dest, importpath, version)

if err != nil {
return err
}

// Copy the module to the destination.
return copyTree(dest, dl.Dir)
}

func copyTree(destRoot, srcRoot string) error {
return filepath.Walk(srcRoot, func(src string, info os.FileInfo, e error) (err error) {
if e != nil {
return e
}
rel, err := filepath.Rel(srcRoot, src)
if err != nil {
return err
}
if rel == "." {
return nil
}
dest := filepath.Join(destRoot, rel)

if info.IsDir() {
return os.Mkdir(dest, 0o777)
} else {
r, err := os.Open(src)
if err != nil {
return err
}
defer r.Close()
w, err := os.Create(dest)
if err != nil {
return err
}
defer func() {
if cerr := w.Close(); err == nil && cerr != nil {
err = cerr
}
}()
_, err = io.Copy(w, r)
return err
}
})
}

// semantic version parsing functions below this point were copied from
// cmd/go/internal/semver and cmd/go/internal/modload at go1.12beta2.

Expand Down
Loading

0 comments on commit cb5dd43

Please sign in to comment.