From 6459f0512c8b380f9dc9566aba6ace5ad05bb8d1 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Tue, 30 Apr 2024 18:13:04 +0200 Subject: [PATCH 1/2] setup-envtest: allow downloading envtest binaries from controller-tools releases --- tools/setup-envtest/README.md | 35 +- tools/setup-envtest/env/env.go | 40 +- tools/setup-envtest/go.mod | 26 +- tools/setup-envtest/go.sum | 102 +-- tools/setup-envtest/main.go | 32 +- tools/setup-envtest/remote/client.go | 211 +----- tools/setup-envtest/remote/gcs_client.go | 198 +++++ tools/setup-envtest/remote/http_client.go | 214 ++++++ tools/setup-envtest/remote/read_body.go | 66 ++ tools/setup-envtest/store/store_test.go | 54 +- tools/setup-envtest/versions/misc_test.go | 16 +- tools/setup-envtest/versions/parse.go | 2 - tools/setup-envtest/versions/platform.go | 50 +- .../setup-envtest/workflows/workflows_test.go | 689 ++++++++++-------- .../workflows/workflows_testutils_test.go | 178 ++++- 15 files changed, 1261 insertions(+), 652 deletions(-) create mode 100644 tools/setup-envtest/remote/gcs_client.go create mode 100644 tools/setup-envtest/remote/http_client.go create mode 100644 tools/setup-envtest/remote/read_body.go diff --git a/tools/setup-envtest/README.md b/tools/setup-envtest/README.md index 1bdeebbc55..11a59c4a1d 100644 --- a/tools/setup-envtest/README.md +++ b/tools/setup-envtest/README.md @@ -2,7 +2,8 @@ This is a small tool that manages binaries for envtest. It can be used to download new binaries, list currently installed and available ones, and -clean up versions. +clean up versions. Binaries can be downloaded either via HTTP via an index +or from GCS. To use it, just go-install it on 1.19+ (it's a separate, self-contained module): @@ -40,6 +41,16 @@ setup-envtest use -i --use-env # sideload a pre-downloaded tarball as Kubernetes 1.16.2 into our store setup-envtest sideload 1.16.2 < downloaded-envtest.tar.gz + +# Per default envtest binaries are downloaded from: +# https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/master/envtest-releases.yaml +# To download from a custom index use the following: +setup-envtest use --index https://custom.com/envtest-releases.yaml + +# To download from the kubebuilder-tools GCS bucket: (default behavior before v0.18) +# Note: This is a Google-owned bucket and it might be shutdown at any time +# see: https://github.com/kubernetes/k8s.io/issues/2647#event-12439345373 +setup-envtest use --use-gcs ``` ## Where does it put all those binaries? @@ -51,16 +62,16 @@ On Linux, this is `$XDG_DATA_HOME`; on Windows, `%LocalAppData`; and on OSX, `~/Library/Application Support`. There's an overall folder that holds all files, and inside that is -a folder for each version/platform pair. The exact directory structure is -not guarnateed, except that the leaf directory will contain the names -expected by envtest. You should always use `setup-envtest fetch` or +a folder for each version/platform pair. The exact directory structure is +not guaranteed, except that the leaf directory will contain the names +expected by envtest. You should always use `setup-envtest fetch` or `setup-envtest switch` (generally with the `-p path` or `-p env` flags) to get the directory that you should use. ## Why do I have to do that `source <(blah blah blah)` thing This is a normal binary, not a shell script, so we can't set the parent -process's environment variables. If you use this by hand a lot and want +process's environment variables. If you use this by hand a lot and want to save the typing, you could put something like the following in your `~/.zshrc` (or similar for bash/fish/whatever, modified to those): @@ -79,7 +90,7 @@ setup-envtest() { There are a few options. First, you'll probably want to set the `-i/--installed` flag. If you want -to avoid forgetting to set this flag, set the `ENVTEST_INSTALLED_ONLY` +to avoid forgetting to set this flag, set the `ENVTEST_INSTALLED_ONLY` env variable, which will switch that flag on by default. Then, you have a few options for managing your binaries: @@ -98,13 +109,17 @@ Then, you have a few options for managing your binaries: `--use-env` on by default. - If you want to use this tool, but download your gziped tarballs - separately, you can use the `sideload` command. You'll need to use the + separately, you can use the `sideload` command. You'll need to use the `-k/--version` flag to indicate which version you're sideloading. After that, it'll be as if you'd installed the binaries with `use`. -- If you want to talk to some internal source, you can use the - `--remote-bucket` and `--remote-server` options. The former sets which +- If you want to talk to some internal source via HTTP, you can simply set `--index` + The index must contain references to envtest binary archives in the same format as: + https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/master/envtest-releases.yaml + +- If you want to talk to some internal source in a GCS "style", you can use the + `--remote-bucket` and `--remote-server` options together with `--use-gcs`. The former sets which GCS bucket to download from, and the latter sets the host to talk to as if it were a GCS endpoint. Theoretically, you could use the latter version to run an internal "mirror" -- the tool expects @@ -114,7 +129,7 @@ Then, you have a few options for managing your binaries: ```json {"items": [ {"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": ""}, - {"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": ""}, + {"name": "kubebuilder-tools-X.Y.Z-os-arch.tar.gz", "md5Hash": ""} ]} ``` diff --git a/tools/setup-envtest/env/env.go b/tools/setup-envtest/env/env.go index e12a107352..dc6a6ea4a2 100644 --- a/tools/setup-envtest/env/env.go +++ b/tools/setup-envtest/env/env.go @@ -33,17 +33,21 @@ type Env struct { // Platform is our current platform Platform versions.PlatformItem - // VerifiySum indicates whether or not we should run checksums. + // VerifySum indicates whether we should run checksums. VerifySum bool - // NoDownload forces us to not contact GCS, looking only - // at local files instead. + // NoDownload forces us to not contact GCS or download the index via HTTP, + // looking only at local files instead. NoDownload bool // ForceDownload forces us to ignore local files and always - // contact GCS & re-download. + // contact GCS or download the index via HTTP & re-download. ForceDownload bool - // Client is our remote client for contacting GCS. - Client *remote.Client + // UseGCS signals if the GCS client is used. + UseGCS bool + + // Client is our remote client for contacting GCS or + // to download the index via HTTP. + Client remote.Client // Log allows us to log. Log logr.Logger @@ -133,7 +137,7 @@ func (e *Env) ListVersions(ctx context.Context) { } // LatestVersion returns the latest version matching our version selector and -// platform from the remote server, with the correspoding checksum for later +// platform from the remote server, with the corresponding checksum for later // use as well. func (e *Env) LatestVersion(ctx context.Context) (versions.Concrete, versions.PlatformItem) { vers, err := e.Client.ListVersions(ctx) @@ -193,7 +197,7 @@ func (e *Env) ExistsAndValid() bool { // // If necessary, it will enumerate on-disk and remote versions to accomplish // this, finding a version that matches our version selector and platform. -// It will always yield a concrete version, it *may* yield a concrete platorm +// It will always yield a concrete version, it *may* yield a concrete platform // as well. func (e *Env) EnsureVersionIsSet(ctx context.Context) { if e.Version.AsConcrete() != nil { @@ -247,13 +251,13 @@ func (e *Env) EnsureVersionIsSet(ctx context.Context) { // if we're not forcing a download, and we have a newer local version, just use that if !e.ForceDownload && localVer != nil && localVer.NewerThan(serverVer) { - e.Platform.Platform = localPlat // update our data with md5 + e.Platform.Platform = localPlat // update our data with hash e.Version.MakeConcrete(*localVer) return } // otherwise, use the new version from the server - e.Platform = platform // update our data with md5 + e.Platform = platform // update our data with hash e.Version.MakeConcrete(serverVer) } @@ -266,13 +270,13 @@ func (e *Env) Fetch(ctx context.Context) { log := e.Log.WithName("fetch") // if we didn't just fetch it, grab the sum to verify - if e.VerifySum && e.Platform.MD5 == "" { + if e.VerifySum && e.Platform.Hash == nil { if err := e.Client.FetchSum(ctx, *e.Version.AsConcrete(), &e.Platform); err != nil { - ExitCause(2, err, "unable to fetch checksum for requested version") + ExitCause(2, err, "unable to fetch hash for requested version") } } if !e.VerifySum { - e.Platform.MD5 = "" // skip verification + e.Platform.Hash = nil // skip verification } var packedPath string @@ -287,7 +291,7 @@ func (e *Env) Fetch(ctx context.Context) { } }) - archiveOut, err := e.FS.TempFile("", "*-"+e.Platform.ArchiveName(*e.Version.AsConcrete())) + archiveOut, err := e.FS.TempFile("", "*-"+e.Platform.ArchiveName(e.UseGCS, *e.Version.AsConcrete())) if err != nil { ExitCause(2, err, "unable to open file to write downloaded archive to") } @@ -365,8 +369,8 @@ func (e *Env) PrintInfo(printFmt PrintFormat) { case PrintOverview: fmt.Fprintf(e.Out, "Version: %s\n", e.Version) fmt.Fprintf(e.Out, "OS/Arch: %s\n", e.Platform) - if e.Platform.MD5 != "" { - fmt.Fprintf(e.Out, "md5: %s\n", e.Platform.MD5) + if e.Platform.Hash != nil { + fmt.Fprintf(e.Out, "%s: %s\n", e.Platform.Hash.Type, e.Platform.Hash.Value) } fmt.Fprintf(e.Out, "Path: %s\n", path) case PrintPath: @@ -409,7 +413,7 @@ func (e *Env) Sideload(ctx context.Context, input io.Reader) { } var ( - // expectedExectuables are the executables that are checked in PathMatches + // expectedExecutables are the executables that are checked in PathMatches // for non-store paths. expectedExecutables = []string{ "kube-apiserver", @@ -458,7 +462,7 @@ func (e *Env) PathMatches(value string) bool { } // versionFromPathName checks if the given path's last component looks like one -// of our versions, and, if so, what version it represents. If succesfull, +// of our versions, and, if so, what version it represents. If successful, // it'll set version and platform, and return true. Otherwise it returns // false. func (e *Env) versionFromPathName(value string) bool { diff --git a/tools/setup-envtest/go.mod b/tools/setup-envtest/go.mod index 3cb53f980d..fa392021d7 100644 --- a/tools/setup-envtest/go.mod +++ b/tools/setup-envtest/go.mod @@ -3,25 +3,27 @@ module sigs.k8s.io/controller-runtime/tools/setup-envtest go 1.22.0 require ( - github.com/go-logr/logr v1.2.4 - github.com/go-logr/zapr v1.2.4 - github.com/onsi/ginkgo/v2 v2.12.1 - github.com/onsi/gomega v1.27.10 + github.com/go-logr/logr v1.4.1 + github.com/go-logr/zapr v1.3.0 + github.com/onsi/ginkgo/v2 v2.17.1 + github.com/onsi/gomega v1.32.0 github.com/spf13/afero v1.6.0 github.com/spf13/pflag v1.0.5 go.uber.org/zap v1.26.0 + k8s.io/apimachinery v0.0.0-20240424173219-03f2f3350dc5 + sigs.k8s.io/yaml v1.3.0 ) require ( github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/text v0.12.0 // indirect - golang.org/x/tools v0.12.0 // indirect - google.golang.org/protobuf v1.28.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.18.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tools/setup-envtest/go.sum b/tools/setup-envtest/go.sum index 1ae5a99838..dd4281ac67 100644 --- a/tools/setup-envtest/go.sum +++ b/tools/setup-envtest/go.sum @@ -1,38 +1,26 @@ -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= -github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -41,72 +29,44 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/apimachinery v0.0.0-20240424173219-03f2f3350dc5 h1:l6ErQDrxBVdvr45UjLjVyvGUwiCRD7A2UF49iYm7ZAc= +k8s.io/apimachinery v0.0.0-20240424173219-03f2f3350dc5/go.mod h1:Xbr0GEGusNQhkPdkN3/WJL9E50/dq40D+fHHqjG+FL8= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/tools/setup-envtest/main.go b/tools/setup-envtest/main.go index 8dca774157..bf4fbdaf3a 100644 --- a/tools/setup-envtest/main.go +++ b/tools/setup-envtest/main.go @@ -49,10 +49,17 @@ var ( binDir = flag.String("bin-dir", "", "directory to store binary assets (default: $OS_SPECIFIC_DATA_DIR/envtest-binaries)") - remoteBucket = flag.String("remote-bucket", "kubebuilder-tools", "remote GCS bucket to download from") + + useGCS = flag.Bool("use-gcs", false, "use GCS to fetch envtest binaries") + + // These flags are only used with --use-gcs. + remoteBucket = flag.String("remote-bucket", "kubebuilder-tools", "remote GCS bucket to download from (only used with --use-gcs)") remoteServer = flag.String("remote-server", "storage.googleapis.com", "remote server to query from. You can override this if you want to run "+ - "an internal storage server instead, or for testing.") + "an internal storage server instead, or for testing. (only used with --use-gcs)") + + // This flag is only used if --use-gcs is not set or false (default). + index = flag.String("index", remote.DefaultIndexURL, "index to discover envtest binaries (only used if --use-gcs is not set, or set to false)") ) // TODO(directxman12): handle interrupts? @@ -81,13 +88,26 @@ func setupEnv(globalLog logr.Logger, version string) *envp.Env { } log.V(1).Info("using binaries directory", "dir", *binDir) - env := &envp.Env{ - Log: globalLog, - Client: &remote.Client{ + var client remote.Client + if useGCS != nil && *useGCS { + client = &remote.GCSClient{ Log: globalLog.WithName("storage-client"), Bucket: *remoteBucket, Server: *remoteServer, - }, + } + log.V(1).Info("using GCS", "bucket", *remoteBucket, "server", *remoteServer) + } else { + client = &remote.HTTPClient{ + Log: globalLog.WithName("storage-client"), + IndexURL: *index, + } + log.V(1).Info("using HTTP", "index", *index) + } + + env := &envp.Env{ + Log: globalLog, + UseGCS: useGCS != nil && *useGCS, + Client: client, VerifySum: *verify, ForceDownload: *force, NoDownload: *installedOnly, diff --git a/tools/setup-envtest/remote/client.go b/tools/setup-envtest/remote/client.go index be82532583..24efd6daff 100644 --- a/tools/setup-envtest/remote/client.go +++ b/tools/setup-envtest/remote/client.go @@ -1,219 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 -// Copyright 2021 The Kubernetes Authors +// Copyright 2024 The Kubernetes Authors package remote import ( "context" - "crypto/md5" //nolint:gosec - "encoding/base64" - "encoding/json" - "errors" - "fmt" "io" - "net/http" - "net/url" - "path" - "sort" - "github.com/go-logr/logr" "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" ) -// objectList is the parts we need of the GCS "list-objects-in-bucket" endpoint. -type objectList struct { - Items []bucketObject `json:"items"` - NextPageToken string `json:"nextPageToken"` -} - -// bucketObject is the parts we need of the GCS object metadata. -type bucketObject struct { - Name string `json:"name"` - Hash string `json:"md5Hash"` -} - -// Client is a basic client for fetching versions of the envtest binary archives -// from GCS. -type Client struct { - // Bucket is the bucket to fetch from. - Bucket string - - // Server is the GCS-like storage server - Server string - - // Log allows us to log. - Log logr.Logger - - // Insecure uses http for testing - Insecure bool -} - -func (c *Client) scheme() string { - if c.Insecure { - return "http" - } - return "https" -} - -// ListVersions lists all available tools versions in the given bucket, along -// with supported os/arch combos and the corresponding hash. -// -// The results are sorted with newer versions first. -func (c *Client) ListVersions(ctx context.Context) ([]versions.Set, error) { - loc := &url.URL{ - Scheme: c.scheme(), - Host: c.Server, - Path: path.Join("/storage/v1/b/", c.Bucket, "o"), - } - query := make(url.Values) - - knownVersions := map[versions.Concrete][]versions.PlatformItem{} - for cont := true; cont; { - c.Log.V(1).Info("listing bucket to get versions", "bucket", c.Bucket) - - loc.RawQuery = query.Encode() - req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) - if err != nil { - return nil, fmt.Errorf("unable to construct request to list bucket items: %w", err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("unable to perform request to list bucket items: %w", err) - } - - err = func() error { - defer resp.Body.Close() - if resp.StatusCode != 200 { - return fmt.Errorf("unable list bucket items -- got status %q from GCS", resp.Status) - } - - var list objectList - if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { - return fmt.Errorf("unable unmarshal bucket items list: %w", err) - } - - // continue listing if needed - cont = list.NextPageToken != "" - query.Set("pageToken", list.NextPageToken) - - for _, item := range list.Items { - ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, item.Name) - if ver == nil { - c.Log.V(1).Info("skipping bucket object -- does not appear to be a versioned tools object", "name", item.Name) - continue - } - c.Log.V(1).Info("found version", "version", ver, "platform", details) - knownVersions[*ver] = append(knownVersions[*ver], versions.PlatformItem{ - Platform: details, - MD5: item.Hash, - }) - } - - return nil - }() - if err != nil { - return nil, err - } - } - - res := make([]versions.Set, 0, len(knownVersions)) - for ver, details := range knownVersions { - res = append(res, versions.Set{Version: ver, Platforms: details}) - } - // sort in inverse order so that the newest one is first - sort.Slice(res, func(i, j int) bool { - first, second := res[i].Version, res[j].Version - return first.NewerThan(second) - }) - - return res, nil -} - -// GetVersion downloads the given concrete version for the given concrete platform, writing it to the out. -func (c *Client) GetVersion(ctx context.Context, version versions.Concrete, platform versions.PlatformItem, out io.Writer) error { - itemName := platform.ArchiveName(version) - loc := &url.URL{ - Scheme: c.scheme(), - Host: c.Server, - Path: path.Join("/storage/v1/b/", c.Bucket, "o", itemName), - RawQuery: "alt=media", - } - - req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) - if err != nil { - return fmt.Errorf("unable to construct request to fetch %s: %w", itemName, err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("unable to fetch %s (%s): %w", itemName, req.URL, err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("unable fetch %s (%s) -- got status %q from GCS", itemName, req.URL, resp.Status) - } - - if platform.MD5 != "" { - // stream in chunks to do the checksum, don't load the whole thing into - // memory to avoid causing issues with big files. - buf := make([]byte, 32*1024) // 32KiB, same as io.Copy - checksum := md5.New() //nolint:gosec - for cont := true; cont; { - amt, err := resp.Body.Read(buf) - if err != nil && !errors.Is(err, io.EOF) { - return fmt.Errorf("unable read next chunk of %s: %w", itemName, err) - } - if amt > 0 { - // checksum never returns errors according to docs - checksum.Write(buf[:amt]) - if _, err := out.Write(buf[:amt]); err != nil { - return fmt.Errorf("unable write next chunk of %s: %w", itemName, err) - } - } - cont = amt > 0 && !errors.Is(err, io.EOF) - } - - sum := base64.StdEncoding.EncodeToString(checksum.Sum(nil)) - - if sum != platform.MD5 { - return fmt.Errorf("checksum mismatch for %s: %s (computed) != %s (reported from GCS)", itemName, sum, platform.MD5) - } - } else if _, err := io.Copy(out, resp.Body); err != nil { - return fmt.Errorf("unable to download %s: %w", itemName, err) - } - return nil -} - -// FetchSum fetches the checksum for the given concrete version & platform into -// the given platform item. -func (c *Client) FetchSum(ctx context.Context, ver versions.Concrete, pl *versions.PlatformItem) error { - itemName := pl.ArchiveName(ver) - loc := &url.URL{ - Scheme: c.scheme(), - Host: c.Server, - Path: path.Join("/storage/v1/b/", c.Bucket, "o", itemName), - } - - req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) - if err != nil { - return fmt.Errorf("unable to construct request to fetch metadata for %s: %w", itemName, err) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("unable to fetch metadata for %s: %w", itemName, err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("unable fetch metadata for %s -- got status %q from GCS", itemName, resp.Status) - } +// Client is an interface to get and list envtest binary archives. +type Client interface { + ListVersions(ctx context.Context) ([]versions.Set, error) - var item bucketObject - if err := json.NewDecoder(resp.Body).Decode(&item); err != nil { - return fmt.Errorf("unable to unmarshal metadata for %s: %w", itemName, err) - } + GetVersion(ctx context.Context, version versions.Concrete, platform versions.PlatformItem, out io.Writer) error - pl.MD5 = item.Hash - return nil + FetchSum(ctx context.Context, ver versions.Concrete, pl *versions.PlatformItem) error } diff --git a/tools/setup-envtest/remote/gcs_client.go b/tools/setup-envtest/remote/gcs_client.go new file mode 100644 index 0000000000..743e9dec64 --- /dev/null +++ b/tools/setup-envtest/remote/gcs_client.go @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2021 The Kubernetes Authors + +package remote + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path" + "sort" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" +) + +// objectList is the parts we need of the GCS "list-objects-in-bucket" endpoint. +type objectList struct { + Items []bucketObject `json:"items"` + NextPageToken string `json:"nextPageToken"` +} + +// bucketObject is the parts we need of the GCS object metadata. +type bucketObject struct { + Name string `json:"name"` + Hash string `json:"md5Hash"` +} + +var _ Client = &GCSClient{} + +// GCSClient is a basic client for fetching versions of the envtest binary archives +// from GCS. +type GCSClient struct { + // Bucket is the bucket to fetch from. + Bucket string + + // Server is the GCS-like storage server + Server string + + // Log allows us to log. + Log logr.Logger + + // Insecure uses http for testing + Insecure bool +} + +func (c *GCSClient) scheme() string { + if c.Insecure { + return "http" + } + return "https" +} + +// ListVersions lists all available tools versions in the given bucket, along +// with supported os/arch combos and the corresponding hash. +// +// The results are sorted with newer versions first. +func (c *GCSClient) ListVersions(ctx context.Context) ([]versions.Set, error) { + loc := &url.URL{ + Scheme: c.scheme(), + Host: c.Server, + Path: path.Join("/storage/v1/b/", c.Bucket, "o"), + } + query := make(url.Values) + + knownVersions := map[versions.Concrete][]versions.PlatformItem{} + for cont := true; cont; { + c.Log.V(1).Info("listing bucket to get versions", "bucket", c.Bucket) + + loc.RawQuery = query.Encode() + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return nil, fmt.Errorf("unable to construct request to list bucket items: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("unable to perform request to list bucket items: %w", err) + } + + err = func() error { + defer resp.Body.Close() + if resp.StatusCode != 200 { + return fmt.Errorf("unable list bucket items -- got status %q from GCS", resp.Status) + } + + var list objectList + if err := json.NewDecoder(resp.Body).Decode(&list); err != nil { + return fmt.Errorf("unable unmarshal bucket items list: %w", err) + } + + // continue listing if needed + cont = list.NextPageToken != "" + query.Set("pageToken", list.NextPageToken) + + for _, item := range list.Items { + ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, item.Name) + if ver == nil { + c.Log.V(1).Info("skipping bucket object -- does not appear to be a versioned tools object", "name", item.Name) + continue + } + c.Log.V(1).Info("found version", "version", ver, "platform", details) + knownVersions[*ver] = append(knownVersions[*ver], versions.PlatformItem{ + Platform: details, + Hash: &versions.Hash{ + Type: versions.MD5HashType, + Encoding: versions.Base64HashEncoding, + Value: item.Hash, + }, + }) + } + + return nil + }() + if err != nil { + return nil, err + } + } + + res := make([]versions.Set, 0, len(knownVersions)) + for ver, details := range knownVersions { + res = append(res, versions.Set{Version: ver, Platforms: details}) + } + // sort in inverse order so that the newest one is first + sort.Slice(res, func(i, j int) bool { + first, second := res[i].Version, res[j].Version + return first.NewerThan(second) + }) + + return res, nil +} + +// GetVersion downloads the given concrete version for the given concrete platform, writing it to the out. +func (c *GCSClient) GetVersion(ctx context.Context, version versions.Concrete, platform versions.PlatformItem, out io.Writer) error { + itemName := platform.ArchiveName(true, version) + loc := &url.URL{ + Scheme: c.scheme(), + Host: c.Server, + Path: path.Join("/storage/v1/b/", c.Bucket, "o", itemName), + RawQuery: "alt=media", + } + + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return fmt.Errorf("unable to construct request to fetch %s: %w", itemName, err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("unable to fetch %s (%s): %w", itemName, req.URL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("unable fetch %s (%s) -- got status %q from GCS", itemName, req.URL, resp.Status) + } + + return readBody(resp, out, itemName, platform) +} + +// FetchSum fetches the checksum for the given concrete version & platform into +// the given platform item. +func (c *GCSClient) FetchSum(ctx context.Context, ver versions.Concrete, pl *versions.PlatformItem) error { + itemName := pl.ArchiveName(true, ver) + loc := &url.URL{ + Scheme: c.scheme(), + Host: c.Server, + Path: path.Join("/storage/v1/b/", c.Bucket, "o", itemName), + } + + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return fmt.Errorf("unable to construct request to fetch metadata for %s: %w", itemName, err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("unable to fetch metadata for %s: %w", itemName, err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("unable fetch metadata for %s -- got status %q from GCS", itemName, resp.Status) + } + + var item bucketObject + if err := json.NewDecoder(resp.Body).Decode(&item); err != nil { + return fmt.Errorf("unable to unmarshal metadata for %s: %w", itemName, err) + } + + pl.Hash = &versions.Hash{ + Type: versions.MD5HashType, + Encoding: versions.Base64HashEncoding, + Value: item.Hash, + } + return nil +} diff --git a/tools/setup-envtest/remote/http_client.go b/tools/setup-envtest/remote/http_client.go new file mode 100644 index 0000000000..0339654a82 --- /dev/null +++ b/tools/setup-envtest/remote/http_client.go @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2021 The Kubernetes Authors + +package remote + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "sort" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" + "sigs.k8s.io/yaml" +) + +// DefaultIndexURL is the default index used in HTTPClient. +var DefaultIndexURL = "https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/HEAD/envtest-releases.yaml" + +var _ Client = &HTTPClient{} + +// HTTPClient is a client for fetching versions of the envtest binary archives +// from an index via HTTP. +type HTTPClient struct { + // Log allows us to log. + Log logr.Logger + + // IndexURL is the URL of the index, defaults to DefaultIndexURL. + IndexURL string +} + +// Index represents an index of envtest binary archives. Example: +// +// releases: +// v1.28.0: +// envtest-v1.28.0-darwin-amd64.tar.gz: +// hash: +// selfLink: +type Index struct { + // Releases maps Kubernetes versions to Releases (envtest archives). + Releases map[string]Release `json:"releases"` +} + +// Release maps an archive name to an archive. +type Release map[string]Archive + +// Archive contains the self link to an archive and its hash. +type Archive struct { + Hash string `json:"hash"` + SelfLink string `json:"selfLink"` +} + +// ListVersions lists all available tools versions in the index, along +// with supported os/arch combos and the corresponding hash. +// +// The results are sorted with newer versions first. +func (c *HTTPClient) ListVersions(ctx context.Context) ([]versions.Set, error) { + index, err := c.getIndex(ctx) + if err != nil { + return nil, err + } + + knownVersions := map[versions.Concrete][]versions.PlatformItem{} + for _, releases := range index.Releases { + for archiveName, archive := range releases { + ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, archiveName) + if ver == nil { + c.Log.V(1).Info("skipping archive -- does not appear to be a versioned tools archive", "name", archiveName) + continue + } + c.Log.V(1).Info("found version", "version", ver, "platform", details) + knownVersions[*ver] = append(knownVersions[*ver], versions.PlatformItem{ + Platform: details, + Hash: &versions.Hash{ + Type: versions.SHA512HashType, + Encoding: versions.HexHashEncoding, + Value: archive.Hash, + }, + }) + } + } + + res := make([]versions.Set, 0, len(knownVersions)) + for ver, details := range knownVersions { + res = append(res, versions.Set{Version: ver, Platforms: details}) + } + // sort in inverse order so that the newest one is first + sort.Slice(res, func(i, j int) bool { + first, second := res[i].Version, res[j].Version + return first.NewerThan(second) + }) + + return res, nil +} + +// GetVersion downloads the given concrete version for the given concrete platform, writing it to the out. +func (c *HTTPClient) GetVersion(ctx context.Context, version versions.Concrete, platform versions.PlatformItem, out io.Writer) error { + index, err := c.getIndex(ctx) + if err != nil { + return err + } + + var loc *url.URL + var name string + for _, releases := range index.Releases { + for archiveName, archive := range releases { + ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, archiveName) + if ver == nil { + c.Log.V(1).Info("skipping archive -- does not appear to be a versioned tools archive", "name", archiveName) + continue + } + + if *ver == version && details.OS == platform.OS && details.Arch == platform.Arch { + loc, err = url.Parse(archive.SelfLink) + if err != nil { + return fmt.Errorf("error parsing selfLink %q, %w", loc, err) + } + name = archiveName + break + } + } + } + if name == "" { + return fmt.Errorf("unable to find archive for %s (%s,%s)", version, platform.OS, platform.Arch) + } + + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return fmt.Errorf("unable to construct request to fetch %s: %w", name, err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("unable to fetch %s (%s): %w", name, req.URL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("unable fetch %s (%s) -- got status %q", name, req.URL, resp.Status) + } + + return readBody(resp, out, name, platform) +} + +// FetchSum fetches the checksum for the given concrete version & platform into +// the given platform item. +func (c *HTTPClient) FetchSum(ctx context.Context, version versions.Concrete, platform *versions.PlatformItem) error { + index, err := c.getIndex(ctx) + if err != nil { + return err + } + + for _, releases := range index.Releases { + for archiveName, archive := range releases { + ver, details := versions.ExtractWithPlatform(versions.ArchiveRE, archiveName) + if ver == nil { + c.Log.V(1).Info("skipping archive -- does not appear to be a versioned tools archive", "name", archiveName) + continue + } + + if *ver == version && details.OS == platform.OS && details.Arch == platform.Arch { + platform.Hash = &versions.Hash{ + Type: versions.SHA512HashType, + Encoding: versions.HexHashEncoding, + Value: archive.Hash, + } + return nil + } + } + } + + return fmt.Errorf("unable to find archive for %s (%s,%s)", version, platform.OS, platform.Arch) +} + +func (c *HTTPClient) getIndex(ctx context.Context) (*Index, error) { + indexURL := c.IndexURL + if indexURL == "" { + indexURL = DefaultIndexURL + } + + loc, err := url.Parse(indexURL) + if err != nil { + return nil, fmt.Errorf("unable to parse index URL: %w", err) + } + + c.Log.V(1).Info("listing versions", "index", indexURL) + + req, err := http.NewRequestWithContext(ctx, "GET", loc.String(), nil) + if err != nil { + return nil, fmt.Errorf("unable to construct request to get index: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, fmt.Errorf("unable to perform request to get index: %w", err) + } + + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unable to get index -- got status %q", resp.Status) + } + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("unable to get index -- unable to read body %w", err) + } + + var index Index + if err := yaml.Unmarshal(responseBody, &index); err != nil { + return nil, fmt.Errorf("unable to unmarshal index: %w", err) + } + return &index, nil +} diff --git a/tools/setup-envtest/remote/read_body.go b/tools/setup-envtest/remote/read_body.go new file mode 100644 index 0000000000..650e41282c --- /dev/null +++ b/tools/setup-envtest/remote/read_body.go @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2024 The Kubernetes Authors + +package remote + +import ( + //nolint:gosec // We're aware that md5 is a weak cryptographic primitive, but we don't have a choice here. + "crypto/md5" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/http" + + "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" +) + +func readBody(resp *http.Response, out io.Writer, archiveName string, platform versions.PlatformItem) error { + if platform.Hash != nil { + // stream in chunks to do the checksum, don't load the whole thing into + // memory to avoid causing issues with big files. + buf := make([]byte, 32*1024) // 32KiB, same as io.Copy + var hasher hash.Hash + switch platform.Hash.Type { + case versions.SHA512HashType: + hasher = sha512.New() + case versions.MD5HashType: + hasher = md5.New() //nolint:gosec // We're aware that md5 is a weak cryptographic primitive, but we don't have a choice here. + default: + return fmt.Errorf("hash type %s not implemented", platform.Hash.Type) + } + for cont := true; cont; { + amt, err := resp.Body.Read(buf) + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("unable read next chunk of %s: %w", archiveName, err) + } + if amt > 0 { + // checksum never returns errors according to docs + hasher.Write(buf[:amt]) + if _, err := out.Write(buf[:amt]); err != nil { + return fmt.Errorf("unable write next chunk of %s: %w", archiveName, err) + } + } + cont = amt > 0 && !errors.Is(err, io.EOF) + } + + var sum string + switch platform.Hash.Encoding { + case versions.Base64HashEncoding: + sum = base64.StdEncoding.EncodeToString(hasher.Sum(nil)) + case versions.HexHashEncoding: + sum = hex.EncodeToString(hasher.Sum(nil)) + default: + return fmt.Errorf("hash encoding %s not implemented", platform.Hash.Encoding) + } + if sum != platform.Hash.Value { + return fmt.Errorf("checksum mismatch for %s: %s (computed) != %s (reported)", archiveName, sum, platform.Hash.Value) + } + } else if _, err := io.Copy(out, resp.Body); err != nil { + return fmt.Errorf("unable to download %s: %w", archiveName, err) + } + return nil +} diff --git a/tools/setup-envtest/store/store_test.go b/tools/setup-envtest/store/store_test.go index d5607aede6..f0d83a1f79 100644 --- a/tools/setup-envtest/store/store_test.go +++ b/tools/setup-envtest/store/store_test.go @@ -125,23 +125,55 @@ var _ = Describe("Store", func() { }) }) - Describe("adding items", func() { + Describe("adding items (GCS archives)", func() { + archiveName := "kubebuilder-tools-1.16.3-linux-amd64.tar.gz" + It("should support .tar.gz input", func() { - Expect(st.Add(logCtx(), newItem, makeFakeArchive(newName))).To(Succeed()) + Expect(st.Add(logCtx(), newItem, makeFakeArchive(archiveName, "kubebuilder/bin/"))).To(Succeed()) Expect(st.Has(newItem)).To(BeTrue(), "should have the item after adding it") }) It("should extract binaries from the given archive to a directly to the item's directory, regardless of path", func() { - Expect(st.Add(logCtx(), newItem, makeFakeArchive(newName))).To(Succeed()) + Expect(st.Add(logCtx(), newItem, makeFakeArchive(archiveName, "kubebuilder/bin/"))).To(Succeed()) dirName := newItem.Platform.BaseName(newItem.Version) - Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "some-file"))).To(HavePrefix(newName + "some-file")) - Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "other-file"))).To(HavePrefix(newName + "other-file")) + Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "some-file"))).To(HavePrefix(archiveName + "some-file")) + Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "other-file"))).To(HavePrefix(archiveName + "other-file")) }) It("should clean up any existing item directory before creating the new one", func() { item := localVersions[0] - Expect(st.Add(logCtx(), item, makeFakeArchive(newName))).To(Succeed()) + Expect(st.Add(logCtx(), item, makeFakeArchive(archiveName, "kubebuilder/bin/"))).To(Succeed()) + Expect(st.Root.Stat(filepath.Join("k8s", item.Platform.BaseName(item.Version)))).NotTo(BeNil(), "new files should exist") + }) + It("should clean up if it errors before finishing", func() { + item := localVersions[0] + Expect(st.Add(logCtx(), item, new(bytes.Buffer))).NotTo(Succeed(), "should fail to extract") + _, err := st.Root.Stat(filepath.Join("k8s", item.Platform.BaseName(item.Version))) + Expect(err).To(HaveOccurred(), "the binaries dir for the item should be gone") + + }) + }) + + Describe("adding items (controller-tools archives)", func() { + archiveName := "envtest-v1.16.3-linux-amd64.tar.gz" + + It("should support .tar.gz input", func() { + Expect(st.Add(logCtx(), newItem, makeFakeArchive(archiveName, "controller-tools/envtest/"))).To(Succeed()) + Expect(st.Has(newItem)).To(BeTrue(), "should have the item after adding it") + }) + + It("should extract binaries from the given archive to a directly to the item's directory, regardless of path", func() { + Expect(st.Add(logCtx(), newItem, makeFakeArchive(archiveName, "controller-tools/envtest/"))).To(Succeed()) + + dirName := newItem.Platform.BaseName(newItem.Version) + Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "some-file"))).To(HavePrefix(archiveName + "some-file")) + Expect(afero.ReadFile(st.Root, filepath.Join("k8s", dirName, "other-file"))).To(HavePrefix(archiveName + "other-file")) + }) + + It("should clean up any existing item directory before creating the new one", func() { + item := localVersions[0] + Expect(st.Add(logCtx(), item, makeFakeArchive(archiveName, "controller-tools/envtest/"))).To(Succeed()) Expect(st.Root.Stat(filepath.Join("k8s", item.Platform.BaseName(item.Version)))).NotTo(BeNil(), "new files should exist") }) It("should clean up if it errors before finishing", func() { @@ -187,8 +219,6 @@ var ( Version: ver(1, 16, 3), Platform: versions.Platform{OS: "linux", Arch: "amd64"}, } - - newName = "kubebuilder-tools-1.16.3-linux-amd64.tar.gz" ) func ver(major, minor, patch int) versions.Concrete { @@ -199,13 +229,13 @@ func ver(major, minor, patch int) versions.Concrete { } } -func makeFakeArchive(magic string) io.Reader { +func makeFakeArchive(magic, relativePath string) io.Reader { out := new(bytes.Buffer) gzipWriter := gzip.NewWriter(out) tarWriter := tar.NewWriter(gzipWriter) Expect(tarWriter.WriteHeader(&tar.Header{ Typeflag: tar.TypeDir, - Name: "kubebuilder/bin/", // so we can ensure we skip non-files + Name: relativePath, // so we can ensure we skip non-files Mode: 0777, })).To(Succeed()) for _, fileName := range []string{"some-file", "other-file"} { @@ -218,9 +248,9 @@ func makeFakeArchive(magic string) io.Reader { panic(err) } - // write to kubebuilder/bin/fileName + // write to relativePath/fileName err := tarWriter.WriteHeader(&tar.Header{ - Name: "kubebuilder/bin/" + fileName, + Name: relativePath + fileName, Size: int64(len(chunk[:])), Mode: 0777, // so we can check that we fix this later }) diff --git a/tools/setup-envtest/versions/misc_test.go b/tools/setup-envtest/versions/misc_test.go index 8a97de0410..dcb87be8b2 100644 --- a/tools/setup-envtest/versions/misc_test.go +++ b/tools/setup-envtest/versions/misc_test.go @@ -95,7 +95,8 @@ var _ = Describe("Platform", func() { Specify("knows how to produce an archive name", func() { plat := Platform{OS: "linux", Arch: "amd64"} ver := Concrete{Major: 1, Minor: 16, Patch: 3} - Expect(plat.ArchiveName(ver)).To(Equal("kubebuilder-tools-1.16.3-linux-amd64.tar.gz")) + Expect(plat.ArchiveName(true, ver)).To(Equal("kubebuilder-tools-1.16.3-linux-amd64.tar.gz")) + Expect(plat.ArchiveName(false, ver)).To(Equal("envtest-v1.16.3-linux-amd64.tar.gz")) }) Describe("parsing", func() { @@ -110,7 +111,7 @@ var _ = Describe("Platform", func() { Expect(ver).To(BeNil()) }) }) - Context("for archive names", func() { + Context("for archive names (GCS)", func() { It("should accept strings of the form kubebuilder-tools-x.y.z-os-arch.tar.gz", func() { ver, plat := ExtractWithPlatform(ArchiveRE, "kubebuilder-tools-1.16.3-linux-amd64.tar.gz") Expect(ver).To(Equal(&Concrete{Major: 1, Minor: 16, Patch: 3})) @@ -121,6 +122,17 @@ var _ = Describe("Platform", func() { Expect(ver).To(BeNil()) }) }) + Context("for archive names (controller-tools)", func() { + It("should accept strings of the form envtest-vx.y.z-os-arch.tar.gz", func() { + ver, plat := ExtractWithPlatform(ArchiveRE, "envtest-v1.16.3-linux-amd64.tar.gz") + Expect(ver).To(Equal(&Concrete{Major: 1, Minor: 16, Patch: 3})) + Expect(plat).To(Equal(Platform{OS: "linux", Arch: "amd64"})) + }) + It("should reject nonsense strings", func() { + ver, _ := ExtractWithPlatform(ArchiveRE, "envtest-v1.16.3-linux-amd64.tar.sum") + Expect(ver).To(BeNil()) + }) + }) }) }) diff --git a/tools/setup-envtest/versions/parse.go b/tools/setup-envtest/versions/parse.go index c053bf8757..21d38bb345 100644 --- a/tools/setup-envtest/versions/parse.go +++ b/tools/setup-envtest/versions/parse.go @@ -17,8 +17,6 @@ var ( // ConcreteVersionRE matches a concrete version anywhere in the string. ConcreteVersionRE = regexp.MustCompile(`(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)`) - // OnlyConcreteVersionRE matches a string that's just a concrete version. - OnlyConcreteVersionRE = regexp.MustCompile(`^` + ConcreteVersionRE.String() + `$`) ) // FromExpr extracts a version from a string in the form of a semver version, diff --git a/tools/setup-envtest/versions/platform.go b/tools/setup-envtest/versions/platform.go index 16c08b38ff..df92bb1b76 100644 --- a/tools/setup-envtest/versions/platform.go +++ b/tools/setup-envtest/versions/platform.go @@ -37,17 +37,57 @@ func (p Platform) BaseName(ver Concrete) string { } // ArchiveName returns the full archive name for this version and platform. -func (p Platform) ArchiveName(ver Concrete) string { - return "kubebuilder-tools-" + p.BaseName(ver) + ".tar.gz" +func (p Platform) ArchiveName(useGCS bool, ver Concrete) string { + if useGCS { + return "kubebuilder-tools-" + p.BaseName(ver) + ".tar.gz" + } + return "envtest-v" + p.BaseName(ver) + ".tar.gz" } // PlatformItem represents a platform with corresponding // known metadata for its download. type PlatformItem struct { Platform - MD5 string + + *Hash } +// Hash of an archive with envtest binaries. +type Hash struct { + // Type of the hash. + // GCS uses MD5HashType, controller-tools uses SHA512HashType. + Type HashType + + // Encoding of the hash value. + // GCS uses Base64HashEncoding, controller-tools uses HexHashEncoding. + Encoding HashEncoding + + // Value of the hash. + Value string +} + +// HashType is the type of a hash. +type HashType string + +const ( + // SHA512HashType represents a sha512 hash + SHA512HashType HashType = "sha512" + + // MD5HashType represents a md5 hash + MD5HashType HashType = "md5" +) + +// HashEncoding is the encoding of a hash +type HashEncoding string + +const ( + // Base64HashEncoding represents base64 encoding + Base64HashEncoding HashEncoding = "base64" + + // HexHashEncoding represents hex encoding + HexHashEncoding HashEncoding = "hex" +) + // Set is a concrete version and all the corresponding platforms that it's available for. type Set struct { Version Concrete @@ -81,5 +121,7 @@ var ( // VersionPlatformRE matches concrete version-platform strings. VersionPlatformRE = regexp.MustCompile(`^` + versionPlatformREBase + `$`) // ArchiveRE matches concrete version-platform.tar.gz strings. - ArchiveRE = regexp.MustCompile(`^kubebuilder-tools-` + versionPlatformREBase + `\.tar\.gz$`) + // The archives published to GCS by kubebuilder use the "kubebuilder-tools-" prefix (e.g. "kubebuilder-tools-1.30.0-darwin-amd64.tar.gz"). + // The archives published to GitHub releases by controller-tools use the "envtest-v" prefix (e.g. "envtest-v1.30.0-darwin-amd64.tar.gz"). + ArchiveRE = regexp.MustCompile(`^(kubebuilder-tools-|envtest-v)` + versionPlatformREBase + `\.tar\.gz$`) ) diff --git a/tools/setup-envtest/workflows/workflows_test.go b/tools/setup-envtest/workflows/workflows_test.go index a0bd7321f7..bae99b8d5b 100644 --- a/tools/setup-envtest/workflows/workflows_test.go +++ b/tools/setup-envtest/workflows/workflows_test.go @@ -5,15 +5,17 @@ package workflows_test import ( "bytes" + "fmt" "io/fs" "path/filepath" + "sort" "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/ghttp" "github.com/spf13/afero" - + "k8s.io/apimachinery/pkg/util/sets" envp "sigs.k8s.io/controller-runtime/tools/setup-envtest/env" "sigs.k8s.io/controller-runtime/tools/setup-envtest/remote" "sigs.k8s.io/controller-runtime/tools/setup-envtest/store" @@ -46,280 +48,288 @@ const ( testStorePath = ".teststore" ) -var _ = Describe("Workflows", func() { - var ( - env *envp.Env - out *bytes.Buffer - server *ghttp.Server - remoteItems []item - ) - BeforeEach(func() { - out = new(bytes.Buffer) - baseFs := afero.Afero{Fs: afero.NewMemMapFs()} - env = &envp.Env{ - Log: testLog, - VerifySum: true, // on by default - FS: baseFs, - Store: &store.Store{Root: afero.NewBasePathFs(baseFs, testStorePath)}, - Out: out, - Platform: versions.PlatformItem{ // default - Platform: versions.Platform{ - OS: "linux", - Arch: "amd64", - }, - }, - Client: &remote.Client{ - Log: testLog.WithName("remote-client"), - Bucket: "kubebuilder-tools-test", // test custom bucket functionality too - Server: "localhost:-1", - Insecure: true, // no https in httptest :-( - }, - } - server = ghttp.NewServer() - env.Client.Server = server.Addr() - - fakeStore(env.FS, testStorePath) - remoteItems = remoteVersions - }) - JustBeforeEach(func() { - handleRemoteVersions(server, remoteItems) - }) - AfterEach(func() { - server.Close() - server = nil - }) +const ( + gcsMode = "GCS" + httpMode = "HTTP" +) + +var _ = Describe("GCS Client", func() { + WorkflowTest(gcsMode) +}) + +var _ = Describe("HTTP Client", func() { + WorkflowTest(httpMode) +}) - Describe("use", func() { - var flow wf.Use +func WorkflowTest(testMode string) { + Describe("Workflows", func() { + var ( + env *envp.Env + out *bytes.Buffer + server *ghttp.Server + remoteGCSItems []item + remoteHTTPItems itemsHTTP + ) BeforeEach(func() { - // some defaults for most tests - env.Version = versions.Spec{ - Selector: ver(1, 16, 0), + out = new(bytes.Buffer) + baseFs := afero.Afero{Fs: afero.NewMemMapFs()} + + server = ghttp.NewServer() + + var client remote.Client + switch testMode { + case gcsMode: + client = &remote.GCSClient{ + Log: testLog.WithName("gcs-client"), + Bucket: "kubebuilder-tools-test", // test custom bucket functionality too + Server: server.Addr(), + Insecure: true, // no https in httptest :-( + } + case httpMode: + client = &remote.HTTPClient{ + Log: testLog.WithName("http-client"), + IndexURL: fmt.Sprintf("http://%s/%s", server.Addr(), "envtest-releases.yaml"), + } } - flow = wf.Use{ - PrintFormat: envp.PrintPath, + + env = &envp.Env{ + Log: testLog, + VerifySum: true, // on by default + FS: baseFs, + Store: &store.Store{Root: afero.NewBasePathFs(baseFs, testStorePath)}, + Out: out, + Platform: versions.PlatformItem{ // default + Platform: versions.Platform{ + OS: "linux", + Arch: "amd64", + }, + }, + Client: client, } - }) - It("should initialize the store if it doesn't exist", func() { - Expect(env.FS.RemoveAll(testStorePath)).To(Succeed()) - // need to set this to a valid remote version cause our store is now empty - env.Version = versions.Spec{Selector: ver(1, 16, 4)} - flow.Do(env) - Expect(env.FS.Stat(testStorePath)).NotTo(BeNil()) + fakeStore(env.FS, testStorePath) + remoteGCSItems = remoteVersionsGCS + remoteHTTPItems = remoteVersionsHTTP + }) + JustBeforeEach(func() { + switch testMode { + case gcsMode: + handleRemoteVersionsGCS(server, remoteGCSItems) + case httpMode: + handleRemoteVersionsHTTP(server, remoteHTTPItems) + } + }) + AfterEach(func() { + server.Close() + server = nil }) - Context("when use env is set", func() { + Describe("use", func() { + var flow wf.Use BeforeEach(func() { - flow.UseEnv = true - }) - It("should fall back to normal behavior when the env is not set", func() { - flow.Do(env) - Expect(out.String()).To(HaveSuffix("/1.16.0-linux-amd64"), "should fall back to a local version") - }) - It("should fall back to normal behavior if binaries are missing", func() { - flow.AssetsPath = ".teststore/missing-binaries" - flow.Do(env) - Expect(out.String()).To(HaveSuffix("/1.16.0-linux-amd64"), "should fall back to a local version") - }) - It("should use the value of the env if it contains the right binaries", func() { - flow.AssetsPath = ".teststore/good-version" - flow.Do(env) - Expect(out.String()).To(Equal(flow.AssetsPath)) - }) - It("should not try and check the version of the binaries", func() { - flow.AssetsPath = ".teststore/wrong-version" - flow.Do(env) - Expect(out.String()).To(Equal(flow.AssetsPath)) + // some defaults for most tests + env.Version = versions.Spec{ + Selector: ver(1, 16, 0), + } + flow = wf.Use{ + PrintFormat: envp.PrintPath, + } }) - It("should not need to contact the network", func() { - server.Close() - flow.AssetsPath = ".teststore/good-version" + + It("should initialize the store if it doesn't exist", func() { + Expect(env.FS.RemoveAll(testStorePath)).To(Succeed()) + // need to set this to a valid remote version cause our store is now empty + env.Version = versions.Spec{Selector: ver(1, 16, 4)} flow.Do(env) - // expect to not get a panic -- if we do, it'll cause the test to fail + Expect(env.FS.Stat(testStorePath)).NotTo(BeNil()) }) - }) - Context("when downloads are disabled", func() { - BeforeEach(func() { - env.NoDownload = true - server.Close() + Context("when use env is set", func() { + BeforeEach(func() { + flow.UseEnv = true + }) + It("should fall back to normal behavior when the env is not set", func() { + flow.Do(env) + Expect(out.String()).To(HaveSuffix("/1.16.0-linux-amd64"), "should fall back to a local version") + }) + It("should fall back to normal behavior if binaries are missing", func() { + flow.AssetsPath = ".teststore/missing-binaries" + flow.Do(env) + Expect(out.String()).To(HaveSuffix("/1.16.0-linux-amd64"), "should fall back to a local version") + }) + It("should use the value of the env if it contains the right binaries", func() { + flow.AssetsPath = ".teststore/good-version" + flow.Do(env) + Expect(out.String()).To(Equal(flow.AssetsPath)) + }) + It("should not try and check the version of the binaries", func() { + flow.AssetsPath = ".teststore/wrong-version" + flow.Do(env) + Expect(out.String()).To(Equal(flow.AssetsPath)) + }) + It("should not need to contact the network", func() { + server.Close() + flow.AssetsPath = ".teststore/good-version" + flow.Do(env) + // expect to not get a panic -- if we do, it'll cause the test to fail + }) }) - // It("should not contact the network") is a gimme here, because we - // call server.Close() above. + Context("when downloads are disabled", func() { + BeforeEach(func() { + env.NoDownload = true + server.Close() + }) - It("should error if no matches are found locally", func() { - defer shouldHaveError() - env.Version.Selector = versions.Concrete{Major: 9001} - flow.Do(env) + // It("should not contact the network") is a gimme here, because we + // call server.Close() above. + + It("should error if no matches are found locally", func() { + defer shouldHaveError() + env.Version.Selector = versions.Concrete{Major: 9001} + flow.Do(env) + }) + It("should settle for the latest local match if latest is requested", func() { + env.Version = versions.Spec{ + CheckLatest: true, + Selector: versions.PatchSelector{ + Major: 1, + Minor: 16, + Patch: versions.AnyPoint, + }, + } + + flow.Do(env) + + // latest on "server" is 1.16.4, shouldn't use that + Expect(out.String()).To(HaveSuffix("/1.16.1-linux-amd64"), "should use the latest local version") + }) + }) + + Context("if latest is requested", func() { + It("should contact the network to see if there's anything newer", func() { + env.Version = versions.Spec{ + CheckLatest: true, + Selector: versions.PatchSelector{ + Major: 1, Minor: 16, Patch: versions.AnyPoint, + }, + } + flow.Do(env) + Expect(out.String()).To(HaveSuffix("/1.16.4-linux-amd64"), "should use the latest remote version") + }) + It("should still use the latest local if the network doesn't have anything newer", func() { + env.Version = versions.Spec{ + CheckLatest: true, + Selector: versions.PatchSelector{ + Major: 1, Minor: 14, Patch: versions.AnyPoint, + }, + } + + flow.Do(env) + + // latest on the server is 1.14.1, latest local is 1.14.26 + Expect(out.String()).To(HaveSuffix("/1.14.26-linux-amd64"), "should use the latest local version") + }) }) - It("should settle for the latest local match if latest is requested", func() { + + It("should check local for a match first", func() { + server.Close() // confirm no network env.Version = versions.Spec{ - CheckLatest: true, - Selector: versions.PatchSelector{ - Major: 1, - Minor: 16, - Patch: versions.AnyPoint, - }, + Selector: versions.TildeSelector{Concrete: ver(1, 16, 0)}, } - flow.Do(env) - - // latest on "server" is 1.16.4, shouldn't use that + // latest on the server is 1.16.4, latest local is 1.16.1 Expect(out.String()).To(HaveSuffix("/1.16.1-linux-amd64"), "should use the latest local version") }) - }) - Context("if latest is requested", func() { - It("should contact the network to see if there's anything newer", func() { + It("should fall back to the network if no local matches are found", func() { env.Version = versions.Spec{ - CheckLatest: true, - Selector: versions.PatchSelector{ - Major: 1, Minor: 16, Patch: versions.AnyPoint, - }, + Selector: versions.TildeSelector{Concrete: ver(1, 19, 0)}, } flow.Do(env) - Expect(out.String()).To(HaveSuffix("/1.16.4-linux-amd64"), "should use the latest remote version") + Expect(out.String()).To(HaveSuffix("/1.19.2-linux-amd64"), "should have a remote version") }) - It("should still use the latest local if the network doesn't have anything newer", func() { + + It("should error out if no matches can be found anywhere", func() { + defer shouldHaveError() env.Version = versions.Spec{ - CheckLatest: true, - Selector: versions.PatchSelector{ - Major: 1, Minor: 14, Patch: versions.AnyPoint, - }, + Selector: versions.TildeSelector{Concrete: ver(0, 0, 1)}, } - flow.Do(env) - - // latest on the server is 1.14.1, latest local is 1.14.26 - Expect(out.String()).To(HaveSuffix("/1.14.26-linux-amd64"), "should use the latest local version") }) - }) - - It("should check local for a match first", func() { - server.Close() // confirm no network - env.Version = versions.Spec{ - Selector: versions.TildeSelector{Concrete: ver(1, 16, 0)}, - } - flow.Do(env) - // latest on the server is 1.16.4, latest local is 1.16.1 - Expect(out.String()).To(HaveSuffix("/1.16.1-linux-amd64"), "should use the latest local version") - }) - - It("should fall back to the network if no local matches are found", func() { - env.Version = versions.Spec{ - Selector: versions.TildeSelector{Concrete: ver(1, 19, 0)}, - } - flow.Do(env) - Expect(out.String()).To(HaveSuffix("/1.19.2-linux-amd64"), "should have a remote version") - }) - - It("should error out if no matches can be found anywhere", func() { - defer shouldHaveError() - env.Version = versions.Spec{ - Selector: versions.TildeSelector{Concrete: ver(0, 0, 1)}, - } - flow.Do(env) - }) - It("should skip local versions matches with non-matching platforms", func() { - env.NoDownload = true // so we get an error - defer shouldHaveError() - env.Version = versions.Spec{ - // has non-matching local versions - Selector: ver(1, 13, 0), - } - - flow.Do(env) - }) - - It("should skip remote version matches with non-matching platforms", func() { - defer shouldHaveError() - env.Version = versions.Spec{ - // has a non-matching remote version - Selector: versions.TildeSelector{Concrete: ver(1, 11, 1)}, - } - flow.Do(env) - }) - - Describe("verifying the checksum", func() { - BeforeEach(func() { - remoteItems = append(remoteItems, item{ - meta: bucketObject{ - Name: "kubebuilder-tools-86.75.309-linux-amd64.tar.gz", - Hash: "nottherightone!", - }, - contents: remoteItems[0].contents, // need a valid tar.gz file to not error from that - }) + It("should skip local versions matches with non-matching platforms", func() { + env.NoDownload = true // so we get an error + defer shouldHaveError() env.Version = versions.Spec{ - Selector: ver(86, 75, 309), + // has non-matching local versions + Selector: ver(1, 13, 0), } + + flow.Do(env) }) - Specify("when enabled, should fail if the downloaded md5 checksum doesn't match", func() { + + It("should skip remote version matches with non-matching platforms", func() { defer shouldHaveError() + env.Version = versions.Spec{ + // has a non-matching remote version + Selector: versions.TildeSelector{Concrete: ver(1, 11, 1)}, + } flow.Do(env) }) - Specify("when disabled, shouldn't check the checksum at all", func() { - env.VerifySum = false - flow.Do(env) + + Describe("verifying the checksum", func() { + BeforeEach(func() { + remoteGCSItems = append(remoteGCSItems, item{ + meta: bucketObject{ + Name: "kubebuilder-tools-86.75.309-linux-amd64.tar.gz", + Hash: "nottherightone!", + }, + contents: remoteGCSItems[0].contents, // need a valid tar.gz file to not error from that + }) + // Recreate remoteHTTPItems to not impact others tests. + remoteHTTPItems = makeContentsHTTP(remoteNamesHTTP) + remoteHTTPItems.index.Releases["v86.75.309"] = map[string]remote.Archive{ + "envtest-v86.75.309-linux-amd64.tar.gz": { + SelfLink: "not used in this test", + Hash: "nottherightone!", + }, + } + // need a valid tar.gz file to not error from that + remoteHTTPItems.contents["envtest-v86.75.309-linux-amd64.tar.gz"] = remoteHTTPItems.contents["envtest-v1.10-darwin-amd64.tar.gz"] + + env.Version = versions.Spec{ + Selector: ver(86, 75, 309), + } + }) + Specify("when enabled, should fail if the downloaded hash doesn't match", func() { + defer shouldHaveError() + flow.Do(env) + }) + Specify("when disabled, shouldn't check the checksum at all", func() { + env.VerifySum = false + flow.Do(env) + }) }) }) - }) - - Describe("list", func() { - // split by fields so we're not matching on whitespace - listFields := func() [][]string { - resLines := strings.Split(strings.TrimSpace(out.String()), "\n") - resFields := make([][]string, len(resLines)) - for i, line := range resLines { - resFields[i] = strings.Fields(line) - } - return resFields - } - Context("when downloads are disabled", func() { - BeforeEach(func() { - server.Close() // ensure no network - env.NoDownload = true - }) - It("should include local contents sorted by version", func() { - env.Version = versions.AnyVersion - env.Platform.Platform = versions.Platform{OS: "*", Arch: "*"} - wf.List{}.Do(env) - - Expect(listFields()).To(Equal([][]string{ - {"(installed)", "v1.17.9", "linux/amd64"}, - {"(installed)", "v1.16.2", "ifonlysingularitywasstillathing/amd64"}, - {"(installed)", "v1.16.2", "linux/yourimagination"}, - {"(installed)", "v1.16.1", "linux/amd64"}, - {"(installed)", "v1.16.0", "linux/amd64"}, - {"(installed)", "v1.14.26", "hyperwarp/pixiedust"}, - {"(installed)", "v1.14.26", "linux/amd64"}, - })) - }) - It("should skip non-matching local contents", func() { - env.Version.Selector = versions.PatchSelector{ - Major: 1, Minor: 16, Patch: versions.AnyPoint, + Describe("list", func() { + // split by fields so we're not matching on whitespace + listFields := func() [][]string { + resLines := strings.Split(strings.TrimSpace(out.String()), "\n") + resFields := make([][]string, len(resLines)) + for i, line := range resLines { + resFields[i] = strings.Fields(line) } - env.Platform.Arch = "*" - wf.List{}.Do(env) - - Expect(listFields()).To(Equal([][]string{ - {"(installed)", "v1.16.2", "linux/yourimagination"}, - {"(installed)", "v1.16.1", "linux/amd64"}, - {"(installed)", "v1.16.0", "linux/amd64"}, - })) + return resFields + } - }) - }) - Context("when downloads are enabled", func() { - Context("when sorting", func() { + Context("when downloads are disabled", func() { BeforeEach(func() { - // shorten the list a bit for expediency - remoteItems = remoteItems[:7] + server.Close() // ensure no network + env.NoDownload = true }) - It("should sort local & remote by version", func() { + It("should include local contents sorted by version", func() { env.Version = versions.AnyVersion env.Platform.Platform = versions.Platform{OS: "*", Arch: "*"} wf.List{}.Do(env) @@ -332,89 +342,160 @@ var _ = Describe("Workflows", func() { {"(installed)", "v1.16.0", "linux/amd64"}, {"(installed)", "v1.14.26", "hyperwarp/pixiedust"}, {"(installed)", "v1.14.26", "linux/amd64"}, - {"(available)", "v1.11.1", "potato/cherrypie"}, - {"(available)", "v1.11.0", "darwin/amd64"}, - {"(available)", "v1.11.0", "linux/amd64"}, - {"(available)", "v1.10.1", "darwin/amd64"}, - {"(available)", "v1.10.1", "linux/amd64"}, })) + }) + It("should skip non-matching local contents", func() { + env.Version.Selector = versions.PatchSelector{ + Major: 1, Minor: 16, Patch: versions.AnyPoint, + } + env.Platform.Arch = "*" + wf.List{}.Do(env) + Expect(listFields()).To(Equal([][]string{ + {"(installed)", "v1.16.2", "linux/yourimagination"}, + {"(installed)", "v1.16.1", "linux/amd64"}, + {"(installed)", "v1.16.0", "linux/amd64"}, + })) }) }) - It("should skip non-matching remote contents", func() { - env.Version.Selector = versions.PatchSelector{ - Major: 1, Minor: 16, Patch: versions.AnyPoint, - } - env.Platform.Arch = "*" - wf.List{}.Do(env) - - Expect(listFields()).To(Equal([][]string{ - {"(installed)", "v1.16.2", "linux/yourimagination"}, - {"(installed)", "v1.16.1", "linux/amd64"}, - {"(installed)", "v1.16.0", "linux/amd64"}, - {"(available)", "v1.16.4", "linux/amd64"}, - })) + Context("when downloads are enabled", func() { + Context("when sorting", func() { + BeforeEach(func() { + // shorten the list a bit for expediency + remoteGCSItems = remoteGCSItems[:7] + + // Recreate remoteHTTPItems to not impact others tests. + remoteHTTPItems = makeContentsHTTP(remoteNamesHTTP) + // Also only keep the first 7 items. + // Get the first 7 archive names + var archiveNames []string + for _, release := range remoteHTTPItems.index.Releases { + for archiveName := range release { + archiveNames = append(archiveNames, archiveName) + } + } + sort.Strings(archiveNames) + archiveNamesSet := sets.Set[string]{}.Insert(archiveNames[:7]...) + // Delete all other archives + for _, release := range remoteHTTPItems.index.Releases { + for archiveName := range release { + if !archiveNamesSet.Has(archiveName) { + delete(release, archiveName) + } + } + } + }) + It("should sort local & remote by version", func() { + env.Version = versions.AnyVersion + env.Platform.Platform = versions.Platform{OS: "*", Arch: "*"} + wf.List{}.Do(env) + + Expect(listFields()).To(Equal([][]string{ + {"(installed)", "v1.17.9", "linux/amd64"}, + {"(installed)", "v1.16.2", "ifonlysingularitywasstillathing/amd64"}, + {"(installed)", "v1.16.2", "linux/yourimagination"}, + {"(installed)", "v1.16.1", "linux/amd64"}, + {"(installed)", "v1.16.0", "linux/amd64"}, + {"(installed)", "v1.14.26", "hyperwarp/pixiedust"}, + {"(installed)", "v1.14.26", "linux/amd64"}, + {"(available)", "v1.11.1", "potato/cherrypie"}, + {"(available)", "v1.11.0", "darwin/amd64"}, + {"(available)", "v1.11.0", "linux/amd64"}, + {"(available)", "v1.10.1", "darwin/amd64"}, + {"(available)", "v1.10.1", "linux/amd64"}, + })) + }) + }) + It("should skip non-matching remote contents", func() { + env.Version.Selector = versions.PatchSelector{ + Major: 1, Minor: 16, Patch: versions.AnyPoint, + } + env.Platform.Arch = "*" + wf.List{}.Do(env) + Expect(listFields()).To(Equal([][]string{ + {"(installed)", "v1.16.2", "linux/yourimagination"}, + {"(installed)", "v1.16.1", "linux/amd64"}, + {"(installed)", "v1.16.0", "linux/amd64"}, + {"(available)", "v1.16.4", "linux/amd64"}, + })) + }) }) }) - }) - Describe("cleanup", func() { - BeforeEach(func() { - server.Close() // ensure no network - flow := wf.Cleanup{} - env.Version = versions.AnyVersion - env.Platform.Arch = "*" - flow.Do(env) - }) + Describe("cleanup", func() { + BeforeEach(func() { + server.Close() // ensure no network + flow := wf.Cleanup{} + env.Version = versions.AnyVersion + env.Platform.Arch = "*" + flow.Do(env) + }) - It("should remove matching versions from the store & keep non-matching ones", func() { - entries, err := env.FS.ReadDir(".teststore/k8s") - Expect(err).NotTo(HaveOccurred(), "should be able to read the store") - Expect(entries).To(ConsistOf( - WithTransform(fs.FileInfo.Name, Equal("1.16.2-ifonlysingularitywasstillathing-amd64")), - WithTransform(fs.FileInfo.Name, Equal("1.14.26-hyperwarp-pixiedust")), - )) + It("should remove matching versions from the store & keep non-matching ones", func() { + entries, err := env.FS.ReadDir(".teststore/k8s") + Expect(err).NotTo(HaveOccurred(), "should be able to read the store") + Expect(entries).To(ConsistOf( + WithTransform(fs.FileInfo.Name, Equal("1.16.2-ifonlysingularitywasstillathing-amd64")), + WithTransform(fs.FileInfo.Name, Equal("1.14.26-hyperwarp-pixiedust")), + )) + }) }) - }) - Describe("sideload", func() { - var ( - flow wf.Sideload - // remote version fake contents are prefixed by the - // name for easier debugging, so we can use that here - expectedPrefix = remoteVersions[0].meta.Name - ) - BeforeEach(func() { - server.Close() // ensure no network - flow.Input = bytes.NewReader(remoteVersions[0].contents) - flow.PrintFormat = envp.PrintPath - }) - It("should initialize the store if it doesn't exist", func() { - env.Version.Selector = ver(1, 10, 0) - Expect(env.FS.RemoveAll(testStorePath)).To(Succeed()) - flow.Do(env) - Expect(env.FS.Stat(testStorePath)).NotTo(BeNil()) - }) - It("should fail if a non-concrete version is given", func() { - defer shouldHaveError() - env.Version = versions.LatestVersion - flow.Do(env) - }) - It("should fail if a non-concrete platform is given", func() { - defer shouldHaveError() - env.Version.Selector = ver(1, 10, 0) - env.Platform.Arch = "*" - flow.Do(env) - }) - It("should load the given gizipped tarball into our store as the given version", func() { - env.Version.Selector = ver(1, 10, 0) - flow.Do(env) - baseName := env.Platform.BaseName(*env.Version.AsConcrete()) - expectedPath := filepath.Join(".teststore/k8s", baseName, "some-file") - outContents, err := env.FS.ReadFile(expectedPath) - Expect(err).NotTo(HaveOccurred(), "should be able to load the unzipped file") - Expect(string(outContents)).To(HavePrefix(expectedPrefix), "should have the debugging prefix") + Describe("sideload", func() { + var ( + flow wf.Sideload + ) + + var expectedPrefix string + if testMode == gcsMode { + // remote version fake contents are prefixed by the + // name for easier debugging, so we can use that here + expectedPrefix = remoteVersionsGCS[0].meta.Name + } + if testMode == httpMode { + // hard coding to one of the archives in remoteVersionsHTTP as we can't pick the "first" of a map. + expectedPrefix = "envtest-v1.10-darwin-amd64.tar.gz" + } + + BeforeEach(func() { + server.Close() // ensure no network + var content []byte + if testMode == gcsMode { + content = remoteVersionsGCS[0].contents + } + if testMode == httpMode { + content = remoteVersionsHTTP.contents[expectedPrefix] + } + flow.Input = bytes.NewReader(content) + flow.PrintFormat = envp.PrintPath + }) + It("should initialize the store if it doesn't exist", func() { + env.Version.Selector = ver(1, 10, 0) + Expect(env.FS.RemoveAll(testStorePath)).To(Succeed()) + flow.Do(env) + Expect(env.FS.Stat(testStorePath)).NotTo(BeNil()) + }) + It("should fail if a non-concrete version is given", func() { + defer shouldHaveError() + env.Version = versions.LatestVersion + flow.Do(env) + }) + It("should fail if a non-concrete platform is given", func() { + defer shouldHaveError() + env.Version.Selector = ver(1, 10, 0) + env.Platform.Arch = "*" + flow.Do(env) + }) + It("should load the given gizipped tarball into our store as the given version", func() { + env.Version.Selector = ver(1, 10, 0) + flow.Do(env) + baseName := env.Platform.BaseName(*env.Version.AsConcrete()) + expectedPath := filepath.Join(".teststore/k8s", baseName, "some-file") + outContents, err := env.FS.ReadFile(expectedPath) + Expect(err).NotTo(HaveOccurred(), "should be able to load the unzipped file") + Expect(string(outContents)).To(HavePrefix(expectedPrefix), "should have the debugging prefix") + }) }) }) -}) +} diff --git a/tools/setup-envtest/workflows/workflows_testutils_test.go b/tools/setup-envtest/workflows/workflows_testutils_test.go index 93e832d763..e796e5d16c 100644 --- a/tools/setup-envtest/workflows/workflows_testutils_test.go +++ b/tools/setup-envtest/workflows/workflows_testutils_test.go @@ -9,7 +9,10 @@ import ( "compress/gzip" "crypto/md5" //nolint:gosec "crypto/rand" + "crypto/sha512" "encoding/base64" + "encoding/hex" + "fmt" "net/http" "path/filepath" @@ -17,12 +20,14 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/ghttp" "github.com/spf13/afero" + "sigs.k8s.io/controller-runtime/tools/setup-envtest/remote" + "sigs.k8s.io/yaml" "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" ) var ( - remoteNames = []string{ + remoteNamesGCS = []string{ "kubebuilder-tools-1.10-darwin-amd64.tar.gz", "kubebuilder-tools-1.10-linux-amd64.tar.gz", "kubebuilder-tools-1.10.1-darwin-amd64.tar.gz", @@ -59,8 +64,68 @@ var ( "kubebuilder-tools-v1.19.2-linux-arm64.tar.gz", "kubebuilder-tools-v1.19.2-linux-ppc64le.tar.gz", } + remoteVersionsGCS = makeContentsGCS(remoteNamesGCS) - remoteVersions = makeContents(remoteNames) + remoteNamesHTTP = remote.Index{ + Releases: map[string]remote.Release{ + "v1.10.0": map[string]remote.Archive{ + "envtest-v1.10-darwin-amd64.tar.gz": {}, + "envtest-v1.10-linux-amd64.tar.gz": {}, + }, + "v1.10.1": map[string]remote.Archive{ + "envtest-v1.10.1-darwin-amd64.tar.gz": {}, + "envtest-v1.10.1-linux-amd64.tar.gz": {}, + }, + "v1.11.0": map[string]remote.Archive{ + "envtest-v1.11.0-darwin-amd64.tar.gz": {}, + "envtest-v1.11.0-linux-amd64.tar.gz": {}, + }, + "v1.11.1": map[string]remote.Archive{ + "envtest-v1.11.1-potato-cherrypie.tar.gz": {}, + }, + "v1.12.3": map[string]remote.Archive{ + "envtest-v1.12.3-darwin-amd64.tar.gz": {}, + "envtest-v1.12.3-linux-amd64.tar.gz": {}, + }, + "v1.13.1": map[string]remote.Archive{ + "envtest-v1.13.1-darwin-amd64.tar.gz": {}, + "envtest-v1.13.1-linux-amd64.tar.gz": {}, + }, + "v1.14.1": map[string]remote.Archive{ + "envtest-v1.14.1-darwin-amd64.tar.gz": {}, + "envtest-v1.14.1-linux-amd64.tar.gz": {}, + }, + "v1.15.5": map[string]remote.Archive{ + "envtest-v1.15.5-darwin-amd64.tar.gz": {}, + "envtest-v1.15.5-linux-amd64.tar.gz": {}, + }, + "v1.16.4": map[string]remote.Archive{ + "envtest-v1.16.4-darwin-amd64.tar.gz": {}, + "envtest-v1.16.4-linux-amd64.tar.gz": {}, + }, + "v1.17.9": map[string]remote.Archive{ + "envtest-v1.17.9-darwin-amd64.tar.gz": {}, + "envtest-v1.17.9-linux-amd64.tar.gz": {}, + }, + "v1.19.0": map[string]remote.Archive{ + "envtest-v1.19.0-darwin-amd64.tar.gz": {}, + "envtest-v1.19.0-linux-amd64.tar.gz": {}, + }, + "v1.19.2": map[string]remote.Archive{ + "envtest-v1.19.2-darwin-amd64.tar.gz": {}, + "envtest-v1.19.2-linux-amd64.tar.gz": {}, + "envtest-v1.19.2-linux-arm64.tar.gz": {}, + "envtest-v1.19.2-linux-ppc64le.tar.gz": {}, + }, + "v1.20.2": map[string]remote.Archive{ + "envtest-v1.20.2-darwin-amd64.tar.gz": {}, + "envtest-v1.20.2-linux-amd64.tar.gz": {}, + "envtest-v1.20.2-linux-arm64.tar.gz": {}, + "envtest-v1.20.2-linux-ppc64le.tar.gz": {}, + }, + }, + } + remoteVersionsHTTP = makeContentsHTTP(remoteNamesHTTP) // keep this sorted. localVersions = []versions.Set{ @@ -100,7 +165,7 @@ type bucketObject struct { Hash string `json:"md5Hash"` } -func makeContents(names []string) []item { +func makeContentsGCS(names []string) []item { res := make([]item, len(names)) for i, name := range names { var chunk [1024 * 48]byte // 1.5 times our chunk read size in GetVersion @@ -108,12 +173,12 @@ func makeContents(names []string) []item { if _, err := rand.Read(chunk[len(name):]); err != nil { panic(err) } - res[i] = verWith(name, chunk[:]) + res[i] = verWithGCS(name, chunk[:]) } return res } -func verWith(name string, contents []byte) item { +func verWithGCS(name string, contents []byte) item { out := new(bytes.Buffer) gzipWriter := gzip.NewWriter(out) tarWriter := tar.NewWriter(gzipWriter) @@ -140,7 +205,7 @@ func verWith(name string, contents []byte) item { return res } -func handleRemoteVersions(server *ghttp.Server, versions []item) { +func handleRemoteVersionsGCS(server *ghttp.Server, versions []item) { list := objectList{Items: make([]bucketObject, len(versions))} for i, ver := range versions { ver := ver // copy to avoid capturing the iteration variable @@ -163,6 +228,107 @@ func handleRemoteVersions(server *ghttp.Server, versions []item) { )) } +type itemsHTTP struct { + index remote.Index + contents map[string][]byte +} + +func makeContentsHTTP(index remote.Index) itemsHTTP { + // This creates a new copy of the index so modifying the index + // in some tests doesn't affect others. + res := itemsHTTP{ + index: remote.Index{ + Releases: map[string]remote.Release{}, + }, + contents: map[string][]byte{}, + } + + for releaseVersion, releases := range index.Releases { + res.index.Releases[releaseVersion] = remote.Release{} + for archiveName := range releases { + var chunk [1024 * 48]byte // 1.5 times our chunk read size in GetVersion + copy(chunk[:], archiveName) + if _, err := rand.Read(chunk[len(archiveName):]); err != nil { + panic(err) + } + content, hash := verWithHTTP(chunk[:]) + + res.index.Releases[releaseVersion][archiveName] = remote.Archive{ + Hash: hash, + // Note: Only storing the name of the archive for now. + // This will be expanded later to a full URL once the server is running. + SelfLink: archiveName, + } + res.contents[archiveName] = content + } + } + return res +} + +func verWithHTTP(contents []byte) ([]byte, string) { + out := new(bytes.Buffer) + gzipWriter := gzip.NewWriter(out) + tarWriter := tar.NewWriter(gzipWriter) + err := tarWriter.WriteHeader(&tar.Header{ + Name: "controller-tools/envtest/some-file", + Size: int64(len(contents)), + Mode: 0777, // so we can check that we fix this later + }) + if err != nil { + panic(err) + } + _, err = tarWriter.Write(contents) + if err != nil { + panic(err) + } + tarWriter.Close() + gzipWriter.Close() + content := out.Bytes() + // controller-tools is using sha512 + hash := sha512.Sum512(content) + hashEncoded := hex.EncodeToString(hash[:]) + return content, hashEncoded +} + +func handleRemoteVersionsHTTP(server *ghttp.Server, items itemsHTTP) { + if server.HTTPTestServer == nil { + // Just return for test cases where server is closed in BeforeEach. Otherwise server.Addr() below panics. + return + } + + // The index from items contains only relative SelfLinks. + // finalIndex will contain the full links based on server.Addr(). + finalIndex := remote.Index{ + Releases: map[string]remote.Release{}, + } + + for releaseVersion, releases := range items.index.Releases { + finalIndex.Releases[releaseVersion] = remote.Release{} + + for archiveName, archive := range releases { + finalIndex.Releases[releaseVersion][archiveName] = remote.Archive{ + Hash: archive.Hash, + SelfLink: fmt.Sprintf("http://%s/%s", server.Addr(), archive.SelfLink), + } + content := items.contents[archiveName] + + // Note: Using the relative path from archive here instead of the full path. + server.RouteToHandler("GET", "/"+archive.SelfLink, func(resp http.ResponseWriter, req *http.Request) { + resp.WriteHeader(http.StatusOK) + Expect(resp.Write(content)).To(Equal(len(content))) + }) + } + } + + indexYAML, err := yaml.Marshal(finalIndex) + Expect(err).ToNot(HaveOccurred()) + + server.RouteToHandler("GET", "/envtest-releases.yaml", ghttp.RespondWith( + http.StatusOK, + indexYAML, + )) +} + func fakeStore(fs afero.Afero, dir string) { By("making the unpacked directory") unpackedBase := filepath.Join(dir, "k8s") From d950bb92291d867339ae42cc9836650b0027f548 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Mon, 6 May 2024 20:39:01 +0200 Subject: [PATCH 2/2] some more deprecations --- tools/setup-envtest/README.md | 9 +++--- tools/setup-envtest/env/env.go | 16 +++++----- tools/setup-envtest/main.go | 32 +++++++++---------- tools/setup-envtest/remote/gcs_client.go | 4 +++ tools/setup-envtest/versions/platform.go | 1 + .../setup-envtest/workflows/workflows_test.go | 2 +- 6 files changed, 35 insertions(+), 29 deletions(-) diff --git a/tools/setup-envtest/README.md b/tools/setup-envtest/README.md index 11a59c4a1d..44505dd8a3 100644 --- a/tools/setup-envtest/README.md +++ b/tools/setup-envtest/README.md @@ -2,8 +2,7 @@ This is a small tool that manages binaries for envtest. It can be used to download new binaries, list currently installed and available ones, and -clean up versions. Binaries can be downloaded either via HTTP via an index -or from GCS. +clean up versions. To use it, just go-install it on 1.19+ (it's a separate, self-contained module): @@ -50,7 +49,8 @@ setup-envtest use --index https://custom.com/envtest-releases.yaml # To download from the kubebuilder-tools GCS bucket: (default behavior before v0.18) # Note: This is a Google-owned bucket and it might be shutdown at any time # see: https://github.com/kubernetes/k8s.io/issues/2647#event-12439345373 -setup-envtest use --use-gcs +# Note: This flag will also be removed soon. +setup-envtest use --use-deprecated-gcs ``` ## Where does it put all those binaries? @@ -119,7 +119,8 @@ Then, you have a few options for managing your binaries: https://raw.githubusercontent.com/kubernetes-sigs/controller-tools/master/envtest-releases.yaml - If you want to talk to some internal source in a GCS "style", you can use the - `--remote-bucket` and `--remote-server` options together with `--use-gcs`. The former sets which + `--remote-bucket` and `--remote-server` options together with `--use-deprecated-gcs`. + Note: This is deprecated and will be removed soon. The former sets which GCS bucket to download from, and the latter sets the host to talk to as if it were a GCS endpoint. Theoretically, you could use the latter version to run an internal "mirror" -- the tool expects diff --git a/tools/setup-envtest/env/env.go b/tools/setup-envtest/env/env.go index dc6a6ea4a2..24857916d7 100644 --- a/tools/setup-envtest/env/env.go +++ b/tools/setup-envtest/env/env.go @@ -26,7 +26,7 @@ import ( // envtest binaries. // // In general, the methods will use the Exit{,Cause} functions from this package -// to indicate errors. Catch them with a `defer HandleExitWithCode()`. +// to indicate errors. Catch them with a `defer HandleExitWithCode()`. type Env struct { // the following *must* be set on input @@ -35,18 +35,18 @@ type Env struct { // VerifySum indicates whether we should run checksums. VerifySum bool - // NoDownload forces us to not contact GCS or download the index via HTTP, + // NoDownload forces us to not contact remote services, // looking only at local files instead. NoDownload bool // ForceDownload forces us to ignore local files and always - // contact GCS or download the index via HTTP & re-download. + // contact remote services & re-download. ForceDownload bool - // UseGCS signals if the GCS client is used. - UseGCS bool + // UseDeprecatedGCS signals if the GCS client is used. + // Note: This will be removed together with remote.GCSClient. + UseDeprecatedGCS bool - // Client is our remote client for contacting GCS or - // to download the index via HTTP. + // Client is our remote client for contacting remote services. Client remote.Client // Log allows us to log. @@ -291,7 +291,7 @@ func (e *Env) Fetch(ctx context.Context) { } }) - archiveOut, err := e.FS.TempFile("", "*-"+e.Platform.ArchiveName(e.UseGCS, *e.Version.AsConcrete())) + archiveOut, err := e.FS.TempFile("", "*-"+e.Platform.ArchiveName(e.UseDeprecatedGCS, *e.Version.AsConcrete())) if err != nil { ExitCause(2, err, "unable to open file to write downloaded archive to") } diff --git a/tools/setup-envtest/main.go b/tools/setup-envtest/main.go index bf4fbdaf3a..7e2761a4f6 100644 --- a/tools/setup-envtest/main.go +++ b/tools/setup-envtest/main.go @@ -50,16 +50,16 @@ var ( binDir = flag.String("bin-dir", "", "directory to store binary assets (default: $OS_SPECIFIC_DATA_DIR/envtest-binaries)") - useGCS = flag.Bool("use-gcs", false, "use GCS to fetch envtest binaries") + useDeprecatedGCS = flag.Bool("use-deprecated-gcs", false, "use GCS to fetch envtest binaries. Note: This is deprecated and will be removed soon. For more details see: https://github.com/kubernetes-sigs/controller-runtime/pull/2811") - // These flags are only used with --use-gcs. - remoteBucket = flag.String("remote-bucket", "kubebuilder-tools", "remote GCS bucket to download from (only used with --use-gcs)") + // These flags are only used with --use-deprecated-gcs. + remoteBucket = flag.String("remote-bucket", "kubebuilder-tools", "remote GCS bucket to download from (only used with --use-deprecated-gcs)") remoteServer = flag.String("remote-server", "storage.googleapis.com", "remote server to query from. You can override this if you want to run "+ - "an internal storage server instead, or for testing. (only used with --use-gcs)") + "an internal storage server instead, or for testing. (only used with --use-deprecated-gcs)") - // This flag is only used if --use-gcs is not set or false (default). - index = flag.String("index", remote.DefaultIndexURL, "index to discover envtest binaries (only used if --use-gcs is not set, or set to false)") + // This flag is only used if --use-deprecated-gcs is not set or false (default). + index = flag.String("index", remote.DefaultIndexURL, "index to discover envtest binaries (only used if --use-deprecated-gcs is not set, or set to false)") ) // TODO(directxman12): handle interrupts? @@ -89,28 +89,28 @@ func setupEnv(globalLog logr.Logger, version string) *envp.Env { log.V(1).Info("using binaries directory", "dir", *binDir) var client remote.Client - if useGCS != nil && *useGCS { - client = &remote.GCSClient{ + if useDeprecatedGCS != nil && *useDeprecatedGCS { + client = &remote.GCSClient{ //nolint:staticcheck // deprecation accepted for now Log: globalLog.WithName("storage-client"), Bucket: *remoteBucket, Server: *remoteServer, } - log.V(1).Info("using GCS", "bucket", *remoteBucket, "server", *remoteServer) + log.V(1).Info("using deprecated GCS client", "bucket", *remoteBucket, "server", *remoteServer) } else { client = &remote.HTTPClient{ Log: globalLog.WithName("storage-client"), IndexURL: *index, } - log.V(1).Info("using HTTP", "index", *index) + log.V(1).Info("using HTTP client", "index", *index) } env := &envp.Env{ - Log: globalLog, - UseGCS: useGCS != nil && *useGCS, - Client: client, - VerifySum: *verify, - ForceDownload: *force, - NoDownload: *installedOnly, + Log: globalLog, + UseDeprecatedGCS: useDeprecatedGCS != nil && *useDeprecatedGCS, + Client: client, + VerifySum: *verify, + ForceDownload: *force, + NoDownload: *installedOnly, Platform: versions.PlatformItem{ Platform: versions.Platform{ OS: *targetOS, diff --git a/tools/setup-envtest/remote/gcs_client.go b/tools/setup-envtest/remote/gcs_client.go index 743e9dec64..85f321d5c5 100644 --- a/tools/setup-envtest/remote/gcs_client.go +++ b/tools/setup-envtest/remote/gcs_client.go @@ -33,6 +33,10 @@ var _ Client = &GCSClient{} // GCSClient is a basic client for fetching versions of the envtest binary archives // from GCS. +// +// Deprecated: This client is deprecated and will be removed soon. +// The kubebuilder GCS bucket that we use with this client might be shutdown at any time, +// see: https://github.com/kubernetes/k8s.io/issues/2647. type GCSClient struct { // Bucket is the bucket to fetch from. Bucket string diff --git a/tools/setup-envtest/versions/platform.go b/tools/setup-envtest/versions/platform.go index df92bb1b76..8b32ccd5bc 100644 --- a/tools/setup-envtest/versions/platform.go +++ b/tools/setup-envtest/versions/platform.go @@ -37,6 +37,7 @@ func (p Platform) BaseName(ver Concrete) string { } // ArchiveName returns the full archive name for this version and platform. +// useGCS is deprecated and will be removed when the remote.GCSClient is removed. func (p Platform) ArchiveName(useGCS bool, ver Concrete) string { if useGCS { return "kubebuilder-tools-" + p.BaseName(ver) + ".tar.gz" diff --git a/tools/setup-envtest/workflows/workflows_test.go b/tools/setup-envtest/workflows/workflows_test.go index bae99b8d5b..8c4007a415 100644 --- a/tools/setup-envtest/workflows/workflows_test.go +++ b/tools/setup-envtest/workflows/workflows_test.go @@ -79,7 +79,7 @@ func WorkflowTest(testMode string) { var client remote.Client switch testMode { case gcsMode: - client = &remote.GCSClient{ + client = &remote.GCSClient{ //nolint:staticcheck // deprecation accepted for now Log: testLog.WithName("gcs-client"), Bucket: "kubebuilder-tools-test", // test custom bucket functionality too Server: server.Addr(),