From 7b34aa33cede11f7c08a9c15b86cd11332725a2c Mon Sep 17 00:00:00 2001 From: Sunil Arora Date: Tue, 8 May 2018 13:15:28 -0700 Subject: [PATCH] Use "dep" to import dependencies for generated project --- cmd/kubebuilder/create/resource/run.go | 3 +- cmd/kubebuilder/initproject/dep_manifest.go | 198 ++++++++++++++++++++ cmd/kubebuilder/initproject/init.go | 5 +- cmd/kubebuilder/initproject/vendor.go | 187 ++++++------------ scripts/generate_dep_manifest.sh | 43 +++++ test.sh | 25 +-- 6 files changed, 303 insertions(+), 158 deletions(-) create mode 100644 cmd/kubebuilder/initproject/dep_manifest.go create mode 100755 scripts/generate_dep_manifest.sh diff --git a/cmd/kubebuilder/create/resource/run.go b/cmd/kubebuilder/create/resource/run.go index cb22a450b5..08965e050c 100644 --- a/cmd/kubebuilder/create/resource/run.go +++ b/cmd/kubebuilder/create/resource/run.go @@ -21,12 +21,13 @@ import ( "log" "os" + "strings" + createutil "github.com/kubernetes-sigs/kubebuilder/cmd/kubebuilder/create/util" generatecmd "github.com/kubernetes-sigs/kubebuilder/cmd/kubebuilder/generate" "github.com/kubernetes-sigs/kubebuilder/cmd/kubebuilder/util" "github.com/markbates/inflect" "github.com/spf13/cobra" - "strings" ) var nonNamespacedKind bool diff --git a/cmd/kubebuilder/initproject/dep_manifest.go b/cmd/kubebuilder/initproject/dep_manifest.go new file mode 100644 index 0000000000..35b359ac90 --- /dev/null +++ b/cmd/kubebuilder/initproject/dep_manifest.go @@ -0,0 +1,198 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package initproject + +// template for dep's manifest file (Gopkg.toml). This is generated using +// scripts/generate_dep_manifest.sh scripts. +const depManifestOverride = ` +[[override]] +name="cloud.google.com/go" +version="v0.21.0" + +[[override]] +name="github.com/PuerkitoBio/purell" +version="v1.1.0" + +[[override]] +name="github.com/PuerkitoBio/urlesc" +revision="de5bf2ad457846296e2031421a34e2568e304e35" + +[[override]] +name="github.com/davecgh/go-spew" +version="v1.1.0" + +[[override]] +name="github.com/emicklei/go-restful" +version="v2.7.0" + +[[override]] +name="github.com/ghodss/yaml" +version="v1.0.0" + +[[override]] +name="github.com/go-openapi/jsonpointer" +revision="3a0015ad55fa9873f41605d3e8f28cd279c32ab2" + +[[override]] +name="github.com/go-openapi/jsonreference" +revision="3fb327e6747da3043567ee86abd02bb6376b6be2" + +[[override]] +name="github.com/go-openapi/spec" +revision="bcff419492eeeb01f76e77d2ebc714dc97b607f5" + +[[override]] +name="github.com/go-openapi/swag" +revision="811b1089cde9dad18d4d0c2d09fbdbf28dbd27a5" + +[[override]] +name="github.com/gogo/protobuf" +version="v1.0.0" + +[[override]] +name="github.com/golang/glog" +revision="23def4e6c14b4da8ac2ed8007337bc5eb5007998" + +[[override]] +name="github.com/golang/groupcache" +revision="66deaeb636dff1ac7d938ce666d090556056a4b0" + +[[override]] +name="github.com/golang/protobuf" +version="v1.1.0" + +[[override]] +name="github.com/google/gofuzz" +revision="24818f796faf91cd76ec7bddd72458fbced7a6c1" + +[[override]] +name="github.com/googleapis/gnostic" +version="v0.1.0" + +[[override]] +name="github.com/hashicorp/golang-lru" +revision="0fb14efe8c47ae851c0034ed7a448854d3d34cf3" + +[[override]] +name="github.com/howeyc/gopass" +revision="bf9dde6d0d2c004a008c27aaee91170c786f6db8" + +[[override]] +name="github.com/imdario/mergo" +version="v0.3.4" + +[[override]] +name="github.com/json-iterator/go" +version="1.1.3" + +[[override]] +name="github.com/mailru/easyjson" +revision="8b799c424f57fa123fc63a99d6383bc6e4c02578" + +[[override]] +name="github.com/modern-go/concurrent" +version="1.0.3" + +[[override]] +name="github.com/modern-go/reflect2" +version="1.0.0" + +[[override]] +name="github.com/onsi/ginkgo" +version="v1.4.0" + +[[override]] +name="github.com/onsi/gomega" +version="v1.3.0" + +[[override]] +name="github.com/spf13/pflag" +version="v1.0.1" + +[[override]] +name="golang.org/x/crypto" +revision="4ec37c66abab2c7e02ae775328b2ff001c3f025a" + +[[override]] +name="golang.org/x/net" +revision="640f4622ab692b87c2f3a94265e6f579fe38263d" + +[[override]] +name="golang.org/x/oauth2" +revision="cdc340f7c179dbbfa4afd43b7614e8fcadde4269" + +[[override]] +name="golang.org/x/sys" +revision="7db1c3b1a98089d0071c84f646ff5c96aad43682" + +[[override]] +name="golang.org/x/text" +version="v0.3.0" + +[[override]] +name="golang.org/x/time" +revision="fbb02b2291d28baffd63558aa44b4b56f178d650" + +[[override]] +name="google.golang.org/appengine" +version="v1.0.0" + +[[override]] +name="gopkg.in/inf.v0" +version="v0.9.1" + +[[override]] +name="gopkg.in/yaml.v2" +version="v2.2.1" + +[[override]] +name="k8s.io/api" +version="kubernetes-1.10.0" + +[[override]] +name="k8s.io/apiextensions-apiserver" +version="kubernetes-1.10.1" + +[[override]] +name="k8s.io/apimachinery" +version="kubernetes-1.10.0" + +[[override]] +name="k8s.io/client-go" +version="kubernetes-1.10.1" + +[[override]] +name="k8s.io/kube-aggregator" +version="kubernetes-1.10.1" + +[[override]] +name="k8s.io/kube-openapi" +revision="f08db293d3ef80052d6513ece19792642a289fea" + +[[override]] +name="sigs.k8s.io/testing_frameworks" +revision="f53464b8b84b4507805a0b033a8377b225163fea" + +[[override]] +name = "github.com/kubernetes-sigs/kubebuilder" +{{ if eq .Version "unknown" -}} +branch="master" +{{ else -}} +version="{{.Version}}" +{{ end }} + +` diff --git a/cmd/kubebuilder/initproject/init.go b/cmd/kubebuilder/initproject/init.go index 2329183364..dc7a0ebb2b 100644 --- a/cmd/kubebuilder/initproject/init.go +++ b/cmd/kubebuilder/initproject/init.go @@ -55,6 +55,9 @@ func runInitRepo(cmd *cobra.Command, args []string) { if versionCmp(version, "go1.10") < 0 { log.Fatalf("The go version is %v, must be 1.10+", version) } + if !depExists() { + log.Fatalf("Dep is not installed. Follow steps at: https://golang.github.io/dep/docs/installation.html") + } if len(domain) == 0 { log.Fatal("Must specify --domain") @@ -62,7 +65,6 @@ func runInitRepo(cmd *cobra.Command, args []string) { cr := util.GetCopyright(copyright) fmt.Printf("Initializing project structure...\n") - RunVendorInstall(nil, nil) if bazel { createBazelWorkspace() } @@ -87,6 +89,7 @@ func runInitRepo(cmd *cobra.Command, args []string) { doInject(cr) doArgs(cr) //os.MkdirAll("bin", 0700) + RunVendorInstall(nil, nil) createBoilerplate() fmt.Printf("Next: Define a resource with:\n" + diff --git a/cmd/kubebuilder/initproject/vendor.go b/cmd/kubebuilder/initproject/vendor.go index 7e83f9fb91..ade3fa811a 100644 --- a/cmd/kubebuilder/initproject/vendor.go +++ b/cmd/kubebuilder/initproject/vendor.go @@ -17,25 +17,27 @@ limitations under the License. package initproject import ( - "archive/tar" - "compress/gzip" "fmt" - "io/ioutil" "log" "os" - "path/filepath" + "os/exec" "github.com/spf13/cobra" + "github.com/kubernetes-sigs/kubebuilder/cmd/kubebuilder/util" "github.com/kubernetes-sigs/kubebuilder/cmd/kubebuilder/version" ) +const ( + depManifestFile = "Gopkg.toml" +) + var vendorInstallCmd = &cobra.Command{ Use: "dep", - Short: "Install Gopkg.toml, Gopkg.lock and vendor/.", - Long: `Install Gopkg.toml, Gopkg.lock and vendor/.`, - Example: `# Bootstrap vendor/ from the src packaged with kubebuilder -kubebuilder init dep + Short: "Install Gopkg.toml and update vendor dependencies.", + Long: `Install Gopkg.toml and update vendor dependencies.`, + Example: `Update the vendor dependencies: +kubebuilder update vendor `, Run: RunVendorInstall, } @@ -43,146 +45,63 @@ kubebuilder init dep var builderCommit string var Update bool -// deleteOld delete all the versions for all packages it is going to untar -func deleteOld() { - // Move up two directories from the location of the `kubebuilder` - // executable to find the `vendor` directory we package with our - // releases. - e, err := os.Executable() - if err != nil { - log.Fatal("unable to get directory of kubebuilder tools") - } - - e = filepath.Dir(filepath.Dir(e)) - - // read the file - f := filepath.Join(e, "bin", "vendor.tar.gz") - fr, err := os.Open(f) - if err != nil { - log.Fatalf("failed to read vendor tar file %s %v", f, err) - } - defer fr.Close() - - // setup gzip of tar - gr, err := gzip.NewReader(fr) - if err != nil { - log.Fatalf("failed to read vendor tar file %s %v", f, err) - } - defer gr.Close() - - // setup tar reader - tr := tar.NewReader(gr) - - for file, err := tr.Next(); err == nil; file, err = tr.Next() { - p := filepath.Join(".", file.Name) - // Delete existing directory first if upgrading - if filepath.Dir(p) != "." { - dir := filepath.Base(filepath.Dir(p)) - parent := filepath.Base(filepath.Dir(filepath.Dir(p))) - gparent := filepath.Base(filepath.Dir(filepath.Dir(filepath.Dir(p)))) - - // Delete the directory if it is a repo or package in a repo - if dir != "vendor" && parent != "vendor" && !(gparent == "vendor" && parent == "github.com") { - os.RemoveAll(filepath.Dir(p)) - } - } - } -} - func RunVendorInstall(cmd *cobra.Command, args []string) { - fmt.Printf("\t%s/\n", filepath.Join("vendor")) - - // Delete old versions of the packages we manage before installing the new ones - if Update { - deleteOld() + if !depExists() { + log.Fatalf("Dep is not installed. Follow steps at: https://golang.github.io/dep/docs/installation.html") } - - // Get the executable directory - e, err := os.Executable() - if err != nil { - log.Fatal("unable to get directory of kubebuilder tools") + var backupFilename string + if Update { + backupFilename = fmt.Sprintf("%s.bkp", depManifestFile) + if err := os.Rename(depManifestFile, backupFilename); err != nil { + fmt.Printf("Error renaming existing Gopkg.toml file: %v \n", err) + return + } } - e = filepath.Dir(e) - - // read the file - f := filepath.Join(e, "vendor.tar.gz") - fr, err := os.Open(f) - if err != nil { - log.Fatalf("failed to read vendor tar file %s %v", f, err) + depTmplArgs := map[string]string{ + "Version": version.GetVersion().KubeBuilderVersion, } - defer fr.Close() - - // setup gzip of tar - gr, err := gzip.NewReader(fr) - if err != nil { - log.Fatalf("failed to read vendor tar file %s %v", f, err) + util.Write(depManifestFile, "dep-manifest-file", depManifestTmpl, depTmplArgs) + if err := runDepEnsure(); err != nil { + fmt.Printf("Error running 'dep ensure': %v\n", err) + fmt.Printf("Previous Gopkg.toml file has been saved at '%s'\n", backupFilename) + return } - defer gr.Close() - - // setup tar reader - tr := tar.NewReader(gr) - - for file, err := tr.Next(); err == nil; file, err = tr.Next() { - if file.FileInfo().IsDir() { - continue - } - p := filepath.Join(".", file.Name) - if Update && filepath.Dir(p) == "." { - continue - } - - err := os.MkdirAll(filepath.Dir(p), 0700) + if Update && backupFilename != "" { + err := os.Remove(backupFilename) if err != nil { - log.Fatalf("Could not create directory %s: %v", filepath.Dir(p), err) - } - b, err := ioutil.ReadAll(tr) - if err != nil { - log.Fatalf("Could not read file %s: %v", file.Name, err) - } - err = ioutil.WriteFile(p, b, os.FileMode(file.Mode)) - if err != nil { - log.Fatalf("Could not write file %s: %v", p, err) + fmt.Printf("Warning: failed to remove backup file: %s", backupFilename) } } + return +} - err = updateDepConfig() +func runDepEnsure() error { + fmt.Printf("Updating vendor dependencies. Running 'dep ensure'....\n") + cmd := exec.Command("dep", "ensure") + o, err := cmd.CombinedOutput() if err != nil { - log.Fatalf("Could not update Gopkg.toml file: %v", err) + fmt.Printf("Failed to run 'dep ensure': %s\n", string(o)) + return err } + fmt.Printf("Updated vendor dependencies successfully.\n") + return nil } -// updateDepConfig updates the Dep config Gopkg.toml to include Kubebuilder -// project dependency. It uses the Kubebuilder version to determine whether -// to include branch or version in the contraint stanza. -func updateDepConfig() error { - var depConstraint string - - kbVersion := version.GetVersion().KubeBuilderVersion - if kbVersion == "unknown" { - // KB is built from master branch - depConstraint = ` -[[constraint]] - branch = "master" - name = "github.com/kubernetes-sigs/kubebuilder" -` - } else { - depConstraint = fmt.Sprintf(` -[[constraint]] - version = "%s" - name = "github.com/kubernetes-sigs/kubebuilder" -`, kbVersion) - } +func depExists() bool { + _, err := exec.LookPath("dep") + return err == nil +} - f, err := os.OpenFile("Gopkg.toml", os.O_APPEND|os.O_WRONLY, 0644) - if err != nil { - return err - } +var depManifestTmpl = fmt.Sprintf("%s\n%s", depManifestHeader, depManifestOverride) - defer f.Close() +const depManifestHeader = ` +# Users add deps lines here - if _, err = f.WriteString(depConstraint); err != nil { - return err - } +[prune] + go-tests = true + #unused-packages = true - return nil -} +# Note: Stanzas below are generated by Kubebuilder and may be rewritten when +# upgrading kubebuilder versions. +# DO NOT MODIFY BELOW THIS LINE. +` diff --git a/scripts/generate_dep_manifest.sh b/scripts/generate_dep_manifest.sh new file mode 100755 index 0000000000..ce412a0413 --- /dev/null +++ b/scripts/generate_dep_manifest.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# +# Script to generate Gopkg.toml file with override stanzas +# Script assumes you have run 'dep ensure' on the project and it is +# in compilable state. +# +# Notes about generating override stanza: +# Create a project using Kubebuilder +# Remove all the override stanzas except the k8s one. +# Run 'dep ensure' to update the dependencies and then +# run this script. +# Save the output of the script and copy its content to replace the value +# depManifestOverride variable in cmd/kubebuilder/initproject/dep_manifest.go +# + +tmp_file="/tmp/dep.txt" +dep status -json|jq '.[]|.ProjectRoot,.Revision, .Version' > $tmp_file +while read name ; do + read revision; read version; + # strip quotes + myver=$(echo "$version" | tr -d '"') + myname=$(echo "$name"|tr -d '"') + if [ "$myname" = "github.com/kubernetes-sigs/kubebuilder" ]; then + continue + fi + if [ "$myver" = "branch master" ] || [ -z "$myver" ]; then + printf "\n[[override]]\nname=$name\nrevision=$revision\n" ; + else + printf "\n[[override]]\nname=$name\nversion=$version\n" ; + fi +done < $tmp_file + +cat << EOF + +[[override]] +name = "github.com/kubernetes-sigs/kubebuilder" +{{ if eq .Version "unknown" -}} +branch = "master" +{{ else -}} +version = "{{.Version}}" +{{ end }} +EOF diff --git a/test.sh b/test.sh index caf1a613c9..56fd2cee7f 100755 --- a/test.sh +++ b/test.sh @@ -74,7 +74,6 @@ rc=0 tmp_root=/tmp kb_root_dir=$tmp_root/kubebuilder -kb_vendor_dir=$tmp_root/vendor kb_orig=$(pwd) # Skip fetching and untaring the tools by setting the SKIP_FETCH_TOOLS variable @@ -127,19 +126,6 @@ function build_kb { go build -o $tmp_root/kubebuilder/bin/kubebuilder-gen ./cmd/kubebuilder-gen } -function prepare_vendor_deps { - header_text "preparing vendor dependencies" - # TODO(droot): clean up this function - rm -rf $kb_vendor_dir && rm -f $tmp_root/Gopkg.toml && rm -f $tmp_root/Gopkg.lock - mkdir -p $kb_vendor_dir/github.com/kubernetes-sigs/kubebuilder/pkg/ || echo "" - cp -r pkg/* $kb_vendor_dir/github.com/kubernetes-sigs/kubebuilder/pkg/ - cp LICENSE $kb_vendor_dir/github.com/kubernetes-sigs/kubebuilder/LICENSE - cp Gopkg.toml Gopkg.lock $tmp_root/ - cp -a vendor/* $kb_vendor_dir/ - cd $tmp_root - tar -czf $kb_root_dir/bin/vendor.tar.gz vendor/ Gopkg.lock Gopkg.toml -} - function prepare_testdir_under_gopath { kb_test_dir=$GOPATH/src/github.com/kubernetes-sigs/kubebuilder-test header_text "preparing test directory $kb_test_dir" @@ -387,7 +373,7 @@ metadata: creationTimestamp: null labels: api: "" - kubebuilder.k8s.io: unknown + kubebuilder.k8s.io: $INJECT_KB_VERSION name: ants.ant.sample.kubernetes.io spec: group: ant.sample.kubernetes.io @@ -422,7 +408,7 @@ metadata: creationTimestamp: null labels: api: "" - kubebuilder.k8s.io: unknown + kubebuilder.k8s.io: $INJECT_KB_VERSION name: bees.insect.sample.kubernetes.io spec: group: insect.sample.kubernetes.io @@ -460,7 +446,7 @@ metadata: creationTimestamp: null labels: api: "" - kubebuilder.k8s.io: unknown + kubebuilder.k8s.io: $INJECT_KB_VERSION name: wasps.insect.sample.kubernetes.io spec: group: insect.sample.kubernetes.io @@ -519,14 +505,9 @@ function test_docs { prepare_staging_dir fetch_tools build_kb -prepare_vendor_deps prepare_testdir_under_gopath generate_crd_resources test_docs test_generated_controller -run_dep_ensure -# Run controller tests after running dep ensure because we want ensure code -# compiles and tests pass after user ran dep ensure on the generated project. -test_generated_controller exit $rc