From e6bc410aa46015b6d4a7ccbb65ebece54b5054ae Mon Sep 17 00:00:00 2001 From: Nicholas Carlson Date: Mon, 6 Mar 2023 16:28:42 -0700 Subject: [PATCH] Fixed failing tests and addressed PR feedback --- go.mod | 16 +-- go.sum | 21 ++-- hack/common.sh | 4 +- pkg/git/fetch.go | 113 ++++++++---------- pkg/git/fetch_test.go | 116 ++++++++++-------- pkg/git/git_keychain.go | 140 +++++----------------- pkg/git/git_keychain_test.go | 178 ++++++++++++++++++---------- pkg/git/k8s_git_keychain.go | 23 ++-- pkg/git/k8s_git_keychain_test.go | 108 ++++++++++------- pkg/git/remote_git_resolver.go | 115 +++++------------- pkg/git/remote_git_resolver_test.go | 37 +++--- pkg/git/resolver.go | 8 +- pkg/git/url_parser.go | 2 +- 13 files changed, 414 insertions(+), 467 deletions(-) diff --git a/go.mod b/go.mod index dcd232456..f08cb3b62 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/buildpacks/imgutil v0.0.0-20220527150729-7a271a852e31 github.com/buildpacks/lifecycle v0.14.1 github.com/ghodss/yaml v1.0.0 - github.com/go-git/go-billy/v5 v5.4.0 github.com/go-git/go-git/v5 v5.6.0 github.com/google/go-cmp v0.5.9 github.com/google/go-containerregistry v0.12.1 @@ -27,7 +26,7 @@ require ( github.com/vdemeester/k8s-pkg-credentialprovider v1.22.4 github.com/whilp/git-urls v1.0.0 go.uber.org/zap v1.23.0 - golang.org/x/crypto v0.3.0 + golang.org/x/crypto v0.7.0 golang.org/x/sync v0.1.0 k8s.io/api v0.24.8 k8s.io/apimachinery v0.24.8 @@ -133,6 +132,7 @@ require ( github.com/fullstorydev/grpcurl v1.8.7 // indirect github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-git/gcfg v1.5.0 // indirect + github.com/go-git/go-billy/v5 v5.4.0 // indirect github.com/go-kit/log v0.2.0 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -294,14 +294,14 @@ require ( go.uber.org/automaxprocs v1.5.1 // indirect go.uber.org/multierr v1.8.0 // indirect golang.org/x/exp v0.0.0-20220823124025-807a23277127 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/net v0.2.0 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/oauth2 v0.1.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/term v0.2.0 // indirect - golang.org/x/text v0.4.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect - golang.org/x/tools v0.2.0 // indirect + golang.org/x/tools v0.6.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/api v0.99.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index b01480664..316d98732 100644 --- a/go.sum +++ b/go.sum @@ -1728,8 +1728,9 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1771,8 +1772,9 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1851,8 +1853,9 @@ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2015,8 +2018,9 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2024,8 +2028,9 @@ golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2035,8 +2040,9 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2132,8 +2138,9 @@ golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpd golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/hack/common.sh b/hack/common.sh index cb953c19c..3b01a2e41 100644 --- a/hack/common.sh +++ b/hack/common.sh @@ -35,7 +35,7 @@ function compile() { completion_image=${IMAGE_PREFIX}completion lifecycle_image=${IMAGE_PREFIX}lifecycle - pack_build ${controller_image} "./cmd/controller" -e BP_GIT2GO_ENABLED=true -e BP_GIT2GO_USE_LIBSSL=true + pack_build ${controller_image} "./cmd/controller" controller_image=${resolved_image_name} pack_build ${build_waiter_image} "./cmd/build-waiter" @@ -44,7 +44,7 @@ function compile() { pack_build ${webhook_image} "./cmd/webhook" webhook_image=${resolved_image_name} - pack_build ${build_init_image} "./cmd/build-init" -e BP_GIT2GO_ENABLED=true -e BP_GIT2GO_USE_LIBSSL=true + pack_build ${build_init_image} "./cmd/build-init" build_init_image=${resolved_image_name} pack_build ${rebase_image} "./cmd/rebase" diff --git a/pkg/git/fetch.go b/pkg/git/fetch.go index 84cc1c317..624ed21d3 100644 --- a/pkg/git/fetch.go +++ b/pkg/git/fetch.go @@ -1,16 +1,20 @@ package git import ( - "github.com/BurntSushi/toml" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" "log" + "net/http" + "net/url" "os" "path" + "time" - // import RemoteConfig + "github.com/BurntSushi/toml" + gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" - + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/client" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/pkg/errors" ) @@ -21,8 +25,11 @@ type Fetcher struct { func (f Fetcher) Fetch(dir, gitURL, gitRevision, metadataDir string) error { f.Logger.Printf("Cloning %q @ %q...", gitURL, gitRevision) + auth, err := f.Keychain.Resolve(gitURL) + if err != nil { + return err + } - // Initialize a repository in the directory using gogit.Init repository, err := gogit.PlainInit(dir, false) if err != nil { return errors.Wrap(err, "initializing repo") @@ -36,44 +43,37 @@ func (f Fetcher) Fetch(dir, gitURL, gitRevision, metadataDir string) error { return errors.Wrap(err, "creating remote") } - err = remote.Fetch(&gogit.FetchOptions{ - RefSpecs: []config.RefSpec{ - config.RefSpec("refs/*:refs/*"), - }, - Auth: nil, - Tags: gogit.AllTags, - RemoteName: defaultRemote, - }) + httpsTransport, err := getHttpsTransport() if err != nil { - return errors.Wrap(err, "fetching remote") + return err } + client.InstallProtocol("https", httpsTransport) - hash, err := resolveRevision(repository, gitRevision) - if err != nil { - return errors.Wrap(err, "resolving revision") + err = remote.Fetch(&gogit.FetchOptions{ + RefSpecs: []config.RefSpec{"refs/*:refs/*"}, + Auth: auth, + }) + if err != nil && err != transport.ErrAuthenticationRequired { + return errors.Wrapf(err, "unable to fetch references for repository") + } else if err == transport.ErrAuthenticationRequired { + return errors.Wrapf(err, "invalid credentials for repository") } - // Look up the commit using the hash - commit, err := repository.CommitObject(*hash) + worktree, err := repository.Worktree() if err != nil { - return errors.Wrap(err, "looking up commit") + return errors.Wrapf(err, "getting worktree for repository") } - worktree, err := repository.Worktree() + hash, err := repository.ResolveRevision(plumbing.Revision(gitRevision)) if err != nil { - return errors.Wrap(err, "getting worktree") + return errors.Wrapf(err, "resolving revision") } - err = worktree.Checkout(&gogit.CheckoutOptions{}) + err = worktree.Checkout(&gogit.CheckoutOptions{Hash: *hash}) if err != nil { - return errors.Wrap(err, "checking out blank") + return errors.Wrapf(err, "checking out revision") } - err = worktree.Checkout(&gogit.CheckoutOptions{ - Hash: plumbing.NewHash(hash.String()), - Create: false, - }) - // Write the git revision to the metadata directory projectMetadataFile, err := os.Create(path.Join(metadataDir, "project-metadata.toml")) if err != nil { @@ -89,7 +89,7 @@ func (f Fetcher) Fetch(dir, gitURL, gitRevision, metadataDir string) error { Revision: gitRevision, }, Version: version{ - Commit: commit.Hash.String(), + Commit: hash.String(), }, }, } @@ -101,42 +101,27 @@ func (f Fetcher) Fetch(dir, gitURL, gitRevision, metadataDir string) error { return nil } -// Implement resolveRevision and return a plumbing.Hash and error -func resolveRevision(repository *gogit.Repository, gitRevision string) (*plumbing.Hash, error) { - ref, err := repository.ResolveRevision(plumbing.Revision(gitRevision)) - if err != nil { - return resolveCommit(gitRevision) - } - return ref, nil -} - -func resolveCommit(gitRevision string) (*plumbing.Hash, error) { - // Use plumbing.NewHash to create a new hash - hash := plumbing.NewHash(gitRevision) - // if hash is empty - if hash == plumbing.ZeroHash { - return nil, errors.Errorf("could not find reference: %s", gitRevision) //invalid hash +func getHttpsTransport() (transport.Transport, error) { + if httpsProxy, exists := os.LookupEnv("HTTPS_PROXY"); exists { + parsedUrl, err := url.Parse(httpsProxy) + if err != nil { + return nil, errors.Wrap(err, "parsing HTTPS_PROXY url") + } + proxyClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyURL(parsedUrl), + }, + Timeout: 15 * time.Second, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + return githttp.NewClient(proxyClient), nil + } else { + return githttp.DefaultClient, nil } - return &hash, nil } -//func resolveRevision(repository *git2go.Repository, gitRevision string) (*git2go.Oid, error) { -// ref, err := repository.References.Dwim(gitRevision) -// if err != nil { -// return resolveCommit(gitRevision) -// } -// -// return ref.Target(), nil -//} - -//func resolveCommit(gitRevision string) (*git2go.Oid, error) { -// oid, err := git2go.NewOid(gitRevision) -// if err != nil { -// return nil, errors.Errorf("could not find reference: %s", gitRevision) //invalid oid -// } -// return oid, nil -//} - type project struct { Source source `toml:"source"` } diff --git a/pkg/git/fetch_test.go b/pkg/git/fetch_test.go index f8d5bc967..eca3eabac 100644 --- a/pkg/git/fetch_test.go +++ b/pkg/git/fetch_test.go @@ -2,22 +2,16 @@ package git import ( "bytes" - "fmt" - "github.com/BurntSushi/toml" - "github.com/go-git/go-billy/v5/osfs" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/storage/filesystem" - "github.com/stretchr/testify/require" - "io/ioutil" "log" "os" "path" "testing" + "github.com/BurntSushi/toml" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/sclevine/spec" + "github.com/stretchr/testify/require" ) func TestGitCheckout(t *testing.T) { @@ -36,14 +30,16 @@ func testGitCheckout(t *testing.T, when spec.G, it spec.S) { it.Before(func() { var err error - testDir, err = ioutil.TempDir("", "test-git") + testDir, err = os.MkdirTemp("", "test-git") require.NoError(t, err) - metadataDir, err = ioutil.TempDir("", "test-git") + metadataDir, err = os.MkdirTemp("", "test-git") require.NoError(t, err) + os.Unsetenv("HTTPS_PROXY") }) it.After(func() { + os.Unsetenv("HTTPS_PROXY") require.NoError(t, os.RemoveAll(testDir)) require.NoError(t, os.RemoveAll(metadataDir)) }) @@ -53,63 +49,85 @@ func testGitCheckout(t *testing.T, when spec.G, it spec.S) { err := fetcher.Fetch(testDir, gitUrl, revision, metadataDir) require.NoError(t, err) - fs := osfs.New(testDir) - storage := filesystem.NewStorage(fs, cache.NewObjectLRUDefault()) - repository, err := gogit.Init(storage, fs) - fmt.Println(outputBuffer.String()) - require.Contains(t, outputBuffer.String(), "Successfully cloned") - branches, err := repository.Branches() + repository, err := gogit.PlainOpen(testDir) require.NoError(t, err) - branches.ForEach(func(branch *plumbing.Reference) error { - fmt.Println("Branch name: ") - fmt.Println(branch.Name()) - return nil - }) + require.Contains(t, outputBuffer.String(), "Successfully cloned") - var projectMetadata project p := path.Join(metadataDir, "project-metadata.toml") - md, err := toml.DecodeFile(p, &projectMetadata) - for k := range md.Keys() { - fmt.Println(k) - } + require.FileExists(t, p) + var projectMetadata project + _, err = toml.DecodeFile(p, &projectMetadata) require.NoError(t, err) require.Equal(t, "git", projectMetadata.Source.Type) require.Equal(t, gitUrl, projectMetadata.Source.Metadata.Repository) require.Equal(t, revision, projectMetadata.Source.Metadata.Revision) - refs, err := repository.References() - refs.ForEach(func(r *plumbing.Reference) error { - fmt.Println(r.Name()) - return nil - }) - + hash, err := repository.ResolveRevision("HEAD") require.NoError(t, err) - //require.Equal(t, ref.Hash(), projectMetadata.Source.Version.Commit) + require.Equal(t, hash.String(), projectMetadata.Source.Version.Commit) } } it("fetches remote HEAD", testFetch("https://github.com/git-fixtures/basic", "master")) - }) -} -type fakeGitKeychain struct { -} + it("fetches a branch", testFetch("https://github.com/git-fixtures/basic", "branch")) + + it("fetches a tag", testFetch("https://github.com/git-fixtures/tags", "lightweight-tag")) -type goGitFakeCredential struct { - GoGitCredential + it("fetches a revision", testFetch("https://github.com/git-fixtures/basic", "b029517f6300c2da0f4b651b8642506cd6aaf45d")) + + it("returns error on non-existent ref", func() { + err := fetcher.Fetch(testDir, "https://github.com/git-fixtures/basic", "doesnotexist", metadataDir) + require.EqualError(t, err, "resolving revision: reference not found") + }) + + it("preserves symbolic links", func() { + err := fetcher.Fetch(testDir, "https://github.com/git-fixtures/symlinks", "master", metadataDir) + require.NoError(t, err) + fileInfo, err := os.Lstat(path.Join(testDir, "bar")) + isSymlink := fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink + require.True(t, isSymlink, "bar is expected to be a symbolic link") + }) + + it("preserves executable permission", func() { + err := fetcher.Fetch(testDir, "https://github.com/pivotal/kpack", "main", metadataDir) + require.NoError(t, err) + + fileInfo, err := os.Lstat(path.Join(testDir, "hack", "apply.sh")) + isExecutable := isExecutableByAll(fileInfo.Mode()) + require.True(t, isExecutable, "apply.sh is expected to be executable by owner, group, and other") + + fileInfo, err = os.Lstat(path.Join(testDir, "hack", "tools.go")) + isExecutable = isExecutableByAny(fileInfo.Mode()) + require.False(t, isExecutable, "tools.go is expected to not be executable") + }) + + it("returns invalid credentials to fetch error on authentication required", func() { + err := fetcher.Fetch(testDir, "git@bitbucket.com:org/repo", "main", metadataDir) + require.ErrorContains(t, err, "unable to fetch references for repository") + }) + + it("uses the http proxy env vars", func() { + require.NoError(t, os.Setenv("HTTPS_PROXY", "http://invalid-proxy")) + defer os.Unsetenv("HTTPS_PROXY") + err := fetcher.Fetch(testDir, "https://github.com/git-fixtures/basic", "master", metadataDir) + require.Error(t, err) + require.Contains(t, err.Error(), "no such host") + }) + }) } -type fakeAuthMethod struct { - transport.AuthMethod +func isExecutableByAny(mode os.FileMode) bool { + return mode&0111 != 0 } -func (c *goGitFakeCredential) Cred() (transport.AuthMethod, error) { - // return fake transport.AuthMethod - return &fakeAuthMethod{}, nil +func isExecutableByAll(mode os.FileMode) bool { + return mode&0111 == 0111 } -func (f fakeGitKeychain) Resolve(url string, usernameFromUrl string, allowedTypes CredentialType) (GoGitCredential, error) { - return &goGitFakeCredential{}, nil - //return nil, errors.New("no auth available") +type fakeGitKeychain struct{} + +func (f fakeGitKeychain) Resolve(gitUrl string) (transport.AuthMethod, error) { + return nil, nil } diff --git a/pkg/git/git_keychain.go b/pkg/git/git_keychain.go index 568b5f0f5..b360e9e2b 100644 --- a/pkg/git/git_keychain.go +++ b/pkg/git/git_keychain.go @@ -1,92 +1,26 @@ package git import ( - "github.com/go-git/go-git/v5/plumbing/transport" - "github.com/go-git/go-git/v5/plumbing/transport/http" - gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "net/url" "sort" "strings" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/pkg/errors" giturls "github.com/whilp/git-urls" - "golang.org/x/crypto/ssh" "github.com/pivotal/kpack/pkg/secret" ) -type CredentialType string - -const ( - CredentialTypeUserpass CredentialType = "userpass" - CredentialTypeSSHKey CredentialType = "sshkey" - CredentialTypeSSHCustom CredentialType = "sshcustom" - CredentialTypeDefault CredentialType = "default" - CredentialTypeUsername CredentialType = "username" - CredentialTypeSSHMemory CredentialType = "sshmemory" -) - -type GoGitCredential interface { - Cred() (transport.AuthMethod, error) -} - -type GoGitSshCredential struct { - GoGitCredential - User string - Signer ssh.Signer - PrivateKey []byte -} - -type GoGitHttpCredential struct { - GoGitCredential - User string - Password string -} - -//func keychainAsCredentialsCallback(gitKeychain GitKeychain) git2go.CredentialsCallback { -// return func(url string, usernameFromUrl string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) { -// cred, err := gitKeychain.Resolve(url, usernameFromUrl, allowedTypes) -// if err != nil { -// return nil, err -// } -// return cred.Cred() -// } -//} - type GitKeychain interface { - Resolve(url string, usernameFromUrl string, allowedTypes CredentialType) (GoGitCredential, error) - //AuthForUrl(url string) (transport.AuthMethod, error) -} - -func (kc *secretGitKeychain) AuthForUrl(url string) (transport.AuthMethod, error) { - cred, err := kc.Resolve(url, "", CredentialTypeSSHKey) - if err != nil { - return nil, err - } - auth, err := cred.Cred() - if err != nil { - return nil, err - } - return auth, nil -} - -func (c *GoGitHttpCredential) Cred() (transport.AuthMethod, error) { - return &http.BasicAuth{ - Username: c.User, - Password: c.Password, - }, nil -} - -func (c *GoGitSshCredential) Cred() (transport.AuthMethod, error) { - signer, err := ssh.ParsePrivateKey([]byte(c.PrivateKey)) - if err != nil { - return nil, err - } - return &gitssh.PublicKeys{User: c.User, Signer: signer}, nil + Resolve(url string) (transport.AuthMethod, error) } type gitCredential interface { - match(host string, allowedTypes CredentialType) bool - goGitCredential(username string) (GoGitCredential, error) + match(url *url.URL) bool + auth() (transport.AuthMethod, error) name() string } @@ -100,16 +34,22 @@ type gitSshAuthCred struct { SecretName string } -func (g gitSshAuthCred) goGitCredential(username string) (GoGitCredential, error) { +func (g gitSshAuthCred) auth() (transport.AuthMethod, error) { sshSecret, err := g.fetchSecret() if err != nil { return nil, err } - return &GoGitSshCredential{ - User: username, - PrivateKey: []byte(sshSecret.PrivateKey), - }, nil + keys, err := gitssh.NewPublicKeys("git", []byte(sshSecret.PrivateKey), "") + if err != nil { + return nil, err + } + + return keys, nil +} + +func (g gitSshAuthCred) match(url *url.URL) bool { + return url.Scheme == "ssh" && gitUrlMatch(url.Host, g.Domain) } func (g gitSshAuthCred) name() string { @@ -122,34 +62,22 @@ type gitBasicAuthCred struct { SecretName string } -func (g gitSshAuthCred) match(host string, allowedTypes CredentialType) bool { - if allowedTypes != CredentialTypeSSHKey { - return false - } - //fmt.Printf("gitSshAuthCred.match: host=%s, g.Domain=%s\n", host, g.Domain) - return gitUrlMatch(host, g.Domain) -} - -func (c gitBasicAuthCred) match(host string, allowedTypes CredentialType) bool { - if allowedTypes != CredentialTypeUserpass { - return false - } - //fmt.Printf("gitSshAuthCred.match: host=%s, g.Domain=%s\n", host, c.Domain) - return gitUrlMatch(host, c.Domain) -} - -func (c gitBasicAuthCred) goGitCredential(_ string) (GoGitCredential, error) { +func (c gitBasicAuthCred) auth() (transport.AuthMethod, error) { basicAuthSecret, err := c.fetchSecret() if err != nil { return nil, err } - return &GoGitHttpCredential{ - User: basicAuthSecret.Username, + return &http.BasicAuth{ + Username: basicAuthSecret.Username, Password: basicAuthSecret.Password, }, nil } +func (c gitBasicAuthCred) match(url *url.URL) bool { + return (url.Scheme == "http" || url.Scheme == "https") && gitUrlMatch(url.Host, c.Domain) +} + func (c gitBasicAuthCred) name() string { return c.SecretName } @@ -191,27 +119,19 @@ func NewMountedSecretGitKeychain(volumeName string, basicAuthSecrets, sshAuthSec }, nil } -// Resolve takes in a URL, username and allowedTypes as input and returns a GoGitCredential that matches the input -func (k *secretGitKeychain) Resolve(url string, username string, allowedTypes CredentialType) (GoGitCredential, error) { - //fmt.Printf("secretGitKeychain::Resolve url->%s, username->%s, allowedTypes->%s\n", url, username, allowedTypes) - u, err := giturls.Parse(url) +func (k *secretGitKeychain) Resolve(rawUrl string) (transport.AuthMethod, error) { + parsedUrl, err := giturls.Parse(rawUrl) if err != nil { return nil, err } - if username == "" { - username = u.User.Username() - } - sort.Slice(k.creds, func(i, j int) bool { return k.creds[i].name() < k.creds[j].name() }) - //fmt.Printf("secretGitKeychain::Resolve number of creds: %d\n", len(k.creds)) for _, cred := range k.creds { - //fmt.Printf("secretGitKeychain::Resolve %s\n", cred.name()) - if cred.match(u.Host, allowedTypes) { - return cred.goGitCredential(username) + if cred.match(parsedUrl) { + return cred.auth() } } - return nil, errors.Errorf("no credentials found for %s", url) + return anonymousAuth, nil } diff --git a/pkg/git/git_keychain_test.go b/pkg/git/git_keychain_test.go index 55bae17e0..c68df21e8 100644 --- a/pkg/git/git_keychain_test.go +++ b/pkg/git/git_keychain_test.go @@ -1,58 +1,82 @@ package git import ( - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/stretchr/testify/require" - "io/ioutil" - corev1 "k8s.io/api/core/v1" "os" "path" - "reflect" "testing" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/sclevine/spec" + "github.com/stretchr/testify/require" + ssh2 "golang.org/x/crypto/ssh" + corev1 "k8s.io/api/core/v1" ) -func TestGitFileKeychain(t *testing.T) { - spec.Run(t, "Test Git Keychain", testGitFileKeychain) +func TestGitKeychain(t *testing.T) { + privateKeyBytes := gitTest{key1: generateRandomPrivateKey(t), key2: generateRandomPrivateKey(t)} + spec.Run(t, "Test Git Keychain", privateKeyBytes.testGitKeychain) +} + +func writeSecrets(testDir string, secrets map[string]map[string][]byte) error { + for name, creds := range secrets { + err := os.MkdirAll(path.Join(testDir, name), 0777) + if err != nil { + return err + } + for k, v := range creds { + err = os.WriteFile(path.Join(testDir, name, k), v, 0600) + if err != nil { + return err + } + } + } + return nil } -func testGitFileKeychain(t *testing.T, when spec.G, it spec.S) { +func (keys gitTest) testGitKeychain(t *testing.T, when spec.G, it spec.S) { var testDir string var keychain GitKeychain it.Before(func() { var err error - testDir, err = ioutil.TempDir("", "git-keychain") + testDir, err = os.MkdirTemp("", "git-keychain") require.NoError(t, err) - require.NoError(t, os.MkdirAll(path.Join(testDir, "github-creds"), 0777)) - require.NoError(t, os.MkdirAll(path.Join(testDir, "more-github-creds"), 0777)) - require.NoError(t, os.MkdirAll(path.Join(testDir, "bitbucket-creds"), 0777)) - require.NoError(t, os.MkdirAll(path.Join(testDir, "basic-bitbucket-creds"), 0777)) - require.NoError(t, os.MkdirAll(path.Join(testDir, "zzz-ssh-bitbucket-creds"), 0777)) - require.NoError(t, os.MkdirAll(path.Join(testDir, "noscheme-creds"), 0777)) - require.NoError(t, os.MkdirAll(path.Join(testDir, "git-ssh-creds"), 0777)) - - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "github-creds", corev1.BasicAuthUsernameKey), []byte("saved-username"), 0600)) - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "github-creds", corev1.BasicAuthPasswordKey), []byte("saved-password"), 0600)) - - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "more-github-creds", corev1.BasicAuthUsernameKey), []byte("another-saved-username"), 0600)) - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "more-github-creds", corev1.BasicAuthPasswordKey), []byte("another-saved-password"), 0600)) - - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "bitbucket-creds", corev1.SSHAuthPrivateKey), []byte("private key 1"), 0600)) - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "zzz-ssh-bitbucket-creds", corev1.SSHAuthPrivateKey), []byte("private key 2"), 0600)) - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "git-ssh-creds", corev1.SSHAuthPrivateKey), []byte("private key 3"), 0600)) - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "basic-bitbucket-creds", corev1.BasicAuthUsernameKey), []byte("saved-username"), 0600)) - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "basic-bitbucket-creds", corev1.BasicAuthPasswordKey), []byte("saved-password"), 0600)) - - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "noscheme-creds", corev1.BasicAuthUsernameKey), []byte("noschemegit-username"), 0600)) - require.NoError(t, ioutil.WriteFile(path.Join(testDir, "noscheme-creds", corev1.BasicAuthPasswordKey), []byte("noschemegit-password"), 0600)) + secrets := map[string]map[string][]byte{ + "github-creds": { + corev1.BasicAuthUsernameKey: []byte("another-saved-username"), + corev1.BasicAuthPasswordKey: []byte("another-saved-password"), + }, + "additional-github-creds": { + corev1.BasicAuthUsernameKey: []byte("saved-username"), + corev1.BasicAuthPasswordKey: []byte("saved-password"), + }, + "bitbucket-creds": { + corev1.SSHAuthPrivateKey: keys.key1, + }, + "basic-bitbucket-creds": { + corev1.BasicAuthUsernameKey: []byte("saved-username"), + corev1.BasicAuthPasswordKey: []byte("saved-password"), + }, + "zzz-ssh-bitbucket-creds": { + corev1.SSHAuthPrivateKey: []byte("private key 2"), + }, + "noscheme-creds": { + corev1.BasicAuthUsernameKey: []byte("noschemegit-username"), + corev1.BasicAuthPasswordKey: []byte("noschemegit-password"), + }, + "git-ssh-creds": { + corev1.SSHAuthPrivateKey: []byte("private key 3"), + }, + } + + require.NoError(t, writeSecrets(testDir, secrets)) keychain, err = NewMountedSecretGitKeychain(testDir, []string{ "github-creds=https://github.com", - "more-github-creds=https://github.com", + "additional-github-creds=https://github.com", "basic-bitbucket-creds=https://bitbucket.com", "noscheme-creds=noschemegit.com"}, []string{ "zzz-ssh-bitbucket-creds=https://bitbucket.com", @@ -66,55 +90,85 @@ func testGitFileKeychain(t *testing.T, when spec.G, it spec.S) { }) when("Resolve", func() { - it("returns alphabetical first git Auth for matching basic auth secrets", func() { - cred, err := keychain.Resolve("https://github.com/org/repo", "", CredentialTypeUserpass) - require.NoError(t, err) - - require.Equal(t, &GoGitHttpCredential{User: "saved-username", Password: "saved-password"}, cred) - gogitCred, err := cred.Cred() - require.NoError(t, err) + when("there are multiple secrets for the same repository", func() { + it("returns alphabetical first git Auth for matching basic auth secrets", func() { + auth, err := keychain.Resolve("https://github.com/org/repo") + require.NoError(t, err) - require.Equal(t, reflect.TypeOf(gogitCred).Elem().String(), reflect.TypeOf(http.BasicAuth{}).String()) + require.Equal(t, &http.BasicAuth{ + Username: "saved-username", + Password: "saved-password", + }, auth) + }) }) - it("returns git Auth for matching secrets without scheme", func() { - cred, err := keychain.Resolve("https://noschemegit.com/org/repo", "", CredentialTypeUserpass) - require.NoError(t, err) + when("there are ssh and basic auth secret types", func() { + it("returns ssh secret if the target is an ssh target", func() { + auth, err := keychain.Resolve("git@bitbucket.com:org/repo") + require.NoError(t, err) - require.Equal(t, &GoGitHttpCredential{User: "noschemegit-username", Password: "noschemegit-password"}, cred) - }) + publicKeys, ok := auth.(*ssh.PublicKeys) + require.True(t, ok) + + require.Equal(t, "git", publicKeys.User) + + expectedSigner, err := ssh2.ParsePrivateKey(keys.key1) + require.NoError(t, err) + require.Equal(t, expectedSigner, publicKeys.Signer) + }) - when("there are ssh and basic auth secret types", func() { it("returns ssh cred for requested ssh credentials", func() { - cred, err := keychain.Resolve("git@bitbucket.com:org/repo", "git", CredentialTypeSSHKey) + auth, err := keychain.Resolve("git@bitbucket.com:org/repo") + require.NoError(t, err) + + _, ok := auth.(*ssh.PublicKeys) + require.True(t, ok) + + signer, err := ssh2.ParsePrivateKey(keys.key1) require.NoError(t, err) - require.Equal(t, &GoGitSshCredential{User: "git", PrivateKey: []byte("private key 1")}, cred) + require.Equal(t, &ssh.PublicKeys{ + User: "git", + Signer: signer, + }, auth) }) - it("returns basic auth secret for requested basic auth credentials", func() { - cred, err := keychain.Resolve("https://bitbucket.com/org/repo", "git", CredentialTypeUserpass) + it("returns basic auth secret if the target is an https target", func() { + auth, err := keychain.Resolve("https://bitbucket.com/org/repo") require.NoError(t, err) - require.Equal(t, &GoGitHttpCredential{User: "saved-username", Password: "saved-password"}, cred) + require.NoError(t, err) + require.Equal(t, &http.BasicAuth{ + Username: "saved-username", + Password: "saved-password", + }, auth) }) }) - it("returns an error if no credentials found", func() { - _, err := keychain.Resolve("https://no-creds-github.com/org/repo", "git", CredentialTypeUserpass) - require.EqualError(t, err, "no credentials found for https://no-creds-github.com/org/repo") + it("returns git Auth for matching basic auth secrets", func() { + auth, err := keychain.Resolve("https://github.com/org/repo") + require.NoError(t, err) + + require.Equal(t, auth, &http.BasicAuth{ + Username: "saved-username", + Password: "saved-password", + }) }) - when("ssh usernameFromUrl is empty during credential callback", func() { - it("determines correct username", func() { - gitKeychain, err := NewMountedSecretGitKeychain(testDir, []string{}, []string{ - "git-ssh-creds=git@my-git-server.com", - }) - cred, err := gitKeychain.Resolve("ssh://git@my-git-server.com/my-org/my-repo.git", "", CredentialTypeSSHKey) - require.NoError(t, err) + it("returns git Auth for matching secrets without scheme", func() { + auth, err := keychain.Resolve("https://noschemegit.com/org/repo") + require.NoError(t, err) - require.Equal(t, &GoGitSshCredential{User: "git", PrivateKey: []byte("private key 3")}, cred) + require.Equal(t, auth, &http.BasicAuth{ + Username: "noschemegit-username", + Password: "noschemegit-password", }) }) + + it("returns anonymous Auth for no matching secret", func() { + auth, err := keychain.Resolve("https://no-creds-github.com/org/repo") + require.NoError(t, err) + require.Nil(t, auth) + }) }) } diff --git a/pkg/git/k8s_git_keychain.go b/pkg/git/k8s_git_keychain.go index bd73f6d54..9bc1b6dc7 100644 --- a/pkg/git/k8s_git_keychain.go +++ b/pkg/git/k8s_git_keychain.go @@ -4,26 +4,32 @@ import ( "context" "fmt" + "github.com/go-git/go-git/v5/plumbing/transport" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" k8sclient "k8s.io/client-go/kubernetes" buildapi "github.com/pivotal/kpack/pkg/apis/build/v1alpha2" + corev1alpha1 "github.com/pivotal/kpack/pkg/apis/core/v1alpha1" "github.com/pivotal/kpack/pkg/secret" ) -type k8sGitKeychainFactory struct { +type k8sGitKeychain struct { secretFetcher secret.Fetcher } -func newK8sGitKeychainFactory(k8sClient k8sclient.Interface) *k8sGitKeychainFactory { - return &k8sGitKeychainFactory{secretFetcher: secret.Fetcher{Client: k8sClient}} +var anonymousAuth transport.AuthMethod = nil + +func newK8sGitKeychain(k8sClient k8sclient.Interface) *k8sGitKeychain { + return &k8sGitKeychain{secretFetcher: secret.Fetcher{Client: k8sClient}} } -func (k *k8sGitKeychainFactory) KeychainForServiceAccount(ctx context.Context, namespace, serviceAccount string) (GitKeychain, error) { +func (k *k8sGitKeychain) Resolve(ctx context.Context, namespace, serviceAccount string, git corev1alpha1.Git) (transport.AuthMethod, error) { secrets, err := k.secretFetcher.SecretsForServiceAccount(ctx, serviceAccount, namespace) if err != nil && !k8serrors.IsNotFound(err) { return nil, err + } else if k8serrors.IsNotFound(err) { + return anonymousAuth, nil } var creds []gitCredential @@ -48,7 +54,7 @@ func (k *k8sGitKeychainFactory) KeychainForServiceAccount(ctx context.Context, n } } - return &secretGitKeychain{creds: creds}, nil + return (&secretGitKeychain{creds: creds}).Resolve(git.URL) } func fetchBasicAuth(s *v1.Secret) func() (secret.BasicAuth, error) { @@ -76,17 +82,10 @@ var matchingDomains = []string{ } func gitUrlMatch(urlMatch, annotatedUrl string) bool { - //fmt.Printf("gitUrlMatch: len(matchingDomains)->%d\n", len(matchingDomains)) for _, format := range matchingDomains { - //fmt.Printf("gitUrlMatch: urlMatch->%s, annotatedUrl->%s, format->%s\n", urlMatch, annotatedUrl, format) - //fmt.Printf("gitUrlMatch: checking match for formatted urlMatch. format->%s, urlMatch->%s, annotatedUrl->%s, Sprintf(format, urlMatch)->%s\n", format, urlMatch, annotatedUrl, fmt.Sprintf(format, urlMatch)) if fmt.Sprintf(format, urlMatch) == annotatedUrl { - // found match for formatted urlMatch - //fmt.Printf("gitUrlMatch: found match for formatted urlMatch. format->%s, urlMatch->%s, annotatedUrl->%s, Sprintf(format, urlMatch)->%s\n", format, urlMatch, annotatedUrl, fmt.Sprintf(format, urlMatch)) return true } } - // no match found for formatted urlMatch - //fmt.Printf("gitUrlMatch: no match found for formatted urlMatch. urlMatch->%s, annotatedUrl->%s\n", urlMatch, annotatedUrl) return false } diff --git a/pkg/git/k8s_git_keychain_test.go b/pkg/git/k8s_git_keychain_test.go index 6c4df18d6..ead5b62f5 100644 --- a/pkg/git/k8s_git_keychain_test.go +++ b/pkg/git/k8s_git_keychain_test.go @@ -6,21 +6,22 @@ import ( "crypto/rsa" "crypto/x509" "encoding/pem" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "reflect" "testing" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/sclevine/spec" "github.com/stretchr/testify/require" + ssh2 "golang.org/x/crypto/ssh" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" buildapi "github.com/pivotal/kpack/pkg/apis/build/v1alpha2" + "github.com/pivotal/kpack/pkg/apis/core/v1alpha1" ) -func Test(t *testing.T) { +func TestK8sGitKeychain(t *testing.T) { privateKeyBytes := gitTest{key1: generateRandomPrivateKey(t), key2: generateRandomPrivateKey(t)} spec.Run(t, "Test Git Keychain", privateKeyBytes.testK8sGitKeychain) } @@ -79,34 +80,34 @@ func (keys gitTest) testK8sGitKeychain(t *testing.T, when spec.G, it spec.S) { }, &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: "secret-4", + Name: "secret-5", Namespace: testNamespace, Annotations: map[string]string{ buildapi.GITSecretAnnotationPrefix: "https://gitlab.com", }, }, - Type: v1.SecretTypeSSHAuth, + Type: v1.SecretTypeBasicAuth, Data: map[string][]byte{ - v1.SSHAuthPrivateKey: keys.key1, + v1.BasicAuthUsernameKey: []byte("gitlab-username"), + v1.BasicAuthPasswordKey: []byte("gitlab-password"), }, }, &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: "secret-5", + Name: "secret-6", Namespace: testNamespace, Annotations: map[string]string{ buildapi.GITSecretAnnotationPrefix: "https://gitlab.com", }, }, - Type: v1.SecretTypeBasicAuth, + Type: v1.SecretTypeSSHAuth, Data: map[string][]byte{ - v1.BasicAuthUsernameKey: []byte("gitlab-username"), - v1.BasicAuthPasswordKey: []byte("gitlab-password"), + v1.SSHAuthPrivateKey: keys.key2, }, }, &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: "secret-6", + Name: "secret-4", Namespace: testNamespace, Annotations: map[string]string{ buildapi.GITSecretAnnotationPrefix: "https://gitlab.com", @@ -114,7 +115,7 @@ func (keys gitTest) testK8sGitKeychain(t *testing.T, when spec.G, it spec.S) { }, Type: v1.SecretTypeSSHAuth, Data: map[string][]byte{ - v1.SSHAuthPrivateKey: keys.key2, + v1.SSHAuthPrivateKey: keys.key1, }, }, &v1.Secret{ @@ -139,68 +140,85 @@ func (keys gitTest) testK8sGitKeychain(t *testing.T, when spec.G, it spec.S) { Secrets: []v1.ObjectReference{ {Name: "secret-1"}, {Name: "secret-2"}, - {Name: "secret-3"}, {Name: "secret-4"}, {Name: "secret-5"}, + {Name: "secret-3"}, {Name: "secret-6"}, {Name: "secret-7"}, }, }) - keychainFactory = newK8sGitKeychainFactory(fakeClient) + keychain = newK8sGitKeychain(fakeClient) ) - when("Keychain resolves", func() { - var keychain GitKeychain + when("K8s Keychain resolves", func() { - it.Before(func() { - var err error - keychain, err = keychainFactory.KeychainForServiceAccount(context.Background(), testNamespace, serviceAccount) + it("returns alphabetical first git Auth for matching secrets with basic auth", func() { + auth, err := keychain.Resolve(context.Background(), testNamespace, serviceAccount, v1alpha1.Git{ + URL: "https://github.com/org/repo", + Revision: "master", + }) require.NoError(t, err) + + require.Equal(t, &http.BasicAuth{ + Username: "saved-username", + Password: "saved-password", + }, auth) }) - it("returns alphabetical first git Auth for matching secrets with basic auth", func() { - cred, err := keychain.Resolve("https://github.com/org/repo", "", CredentialTypeUserpass) + it("returns the alphabetical first secretRef for ssh auth", func() { + auth, err := keychain.Resolve(context.Background(), testNamespace, serviceAccount, v1alpha1.Git{ + URL: "git@gitlab.com:org/repo", + Revision: "master", + }) require.NoError(t, err) - require.Equal(t, GoGitHttpCredential{ - User: "saved-username", - Password: "saved-password", - }, cred) + publicKeys, ok := auth.(*ssh.PublicKeys) + require.True(t, ok) - gogitCred, err := cred.Cred() - require.NoError(t, err) + require.Equal(t, "git", publicKeys.User) - require.Equal(t, reflect.TypeOf(gogitCred).Elem().String(), reflect.TypeOf(http.BasicAuth{}).String()) + expectedSigner, err := ssh2.ParsePrivateKey(keys.key1) + require.NoError(t, err) + require.Equal(t, expectedSigner, publicKeys.Signer) }) - it("returns the alphabetical first secretRef for ssh auth", func() { - cred, err := keychain.Resolve("https://gitlab.com/my-repo.git", "gituser", CredentialTypeSSHKey) + it("returns git Auth for matching secrets with ssh auth", func() { + auth, err := keychain.Resolve(context.Background(), testNamespace, serviceAccount, v1alpha1.Git{ + URL: "git@bitbucket.com:org/repo", + Revision: "master", + }) require.NoError(t, err) - require.Equal(t, &GoGitSshCredential{ - User: "gituser", - PrivateKey: keys.key1, - }, cred) + publicKeys, ok := auth.(*ssh.PublicKeys) + require.True(t, ok) - gogitCred, err := cred.Cred() - require.NoError(t, err) + require.Equal(t, "git", publicKeys.User) - require.Equal(t, reflect.TypeOf(gogitCred).Elem().String(), reflect.TypeOf(ssh.PublicKeys{}).String()) + signer, err := ssh2.ParsePrivateKey(keys.key1) + require.NoError(t, err) + require.Equal(t, signer, publicKeys.Signer) }) it("returns git Auth for matching secrets without scheme", func() { - cred, err := keychain.Resolve("https://noschemegit.com/org/repo", "", CredentialTypeUserpass) + auth, err := keychain.Resolve(context.Background(), testNamespace, serviceAccount, v1alpha1.Git{ + URL: "https://noschemegit.com/org/repo", + Revision: "master", + }) require.NoError(t, err) - require.Equal(t, GoGitHttpCredential{ - User: "noschemegit-username", + require.Equal(t, &http.BasicAuth{ + Username: "noschemegit-username", Password: "noschemegit-password", - }, cred) + }, auth) }) - it("returns an error if no credential are found", func() { - _, err := keychain.Resolve("https://notfound.com/org/repo", "git", CredentialTypeUserpass) - require.EqualError(t, err, "no credentials found for https://notfound.com/org/repo") + it("returns anonymous Auth for no matching secret", func() { + auth, err := keychain.Resolve(context.Background(), testNamespace, serviceAccount, v1alpha1.Git{ + URL: "https://no-creds-github.com/org/repo", + Revision: "master", + }) + require.NoError(t, err) + require.Nil(t, auth) }) }) } diff --git a/pkg/git/remote_git_resolver.go b/pkg/git/remote_git_resolver.go index 7b6fa9ec8..b9aad4c44 100644 --- a/pkg/git/remote_git_resolver.go +++ b/pkg/git/remote_git_resolver.go @@ -1,61 +1,27 @@ package git import ( - "fmt" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport" - "io/ioutil" - "log" - "os" - - "github.com/pkg/errors" + "github.com/go-git/go-git/v5/plumbing/transport/client" + "github.com/go-git/go-git/v5/storage/memory" corev1alpha1 "github.com/pivotal/kpack/pkg/apis/core/v1alpha1" ) const defaultRemote = "origin" -var discardLogger = log.New(ioutil.Discard, "", 0) - -type remoteGitResolver struct { -} +type remoteGitResolver struct{} -func (*remoteGitResolver) Resolve(keychain GitKeychain, sourceConfig corev1alpha1.SourceConfig) (corev1alpha1.ResolvedSourceConfig, error) { - // initialize a new repository in a temporary directory - dir, err := ioutil.TempDir("", "kpack-git") - if err != nil { - return corev1alpha1.ResolvedSourceConfig{}, errors.Wrap(err, "creating temp dir") - } - defer os.RemoveAll(dir) - // initialize a new repository - repository, err := gogit.PlainInit(dir, false) - if err != nil { - return corev1alpha1.ResolvedSourceConfig{}, errors.Wrap(err, "initializing repo") - } - // create a new remote - remote, err := repository.CreateRemote(&config.RemoteConfig{ +func (*remoteGitResolver) Resolve(auth transport.AuthMethod, sourceConfig corev1alpha1.SourceConfig) (corev1alpha1.ResolvedSourceConfig, error) { + remote := gogit.NewRemote(memory.NewStorage(), &config.RemoteConfig{ Name: defaultRemote, URLs: []string{sourceConfig.Git.URL}, }) - cred, err := keychain.Resolve(sourceConfig.Git.URL, "", CredentialTypeUserpass) - if err != nil { - return corev1alpha1.ResolvedSourceConfig{}, errors.Wrap(err, "getting auth for url") - } - - auth, err := cred.Cred() - if err != nil { - return corev1alpha1.ResolvedSourceConfig{}, errors.Wrap(err, "getting auth for url") - } - - // fetch the remote - err = remote.Fetch(&gogit.FetchOptions{ - RemoteName: defaultRemote, - Auth: auth, - Progress: discardLogger.Writer(), - }) + httpsTransport, err := getHttpsTransport() if err != nil { return corev1alpha1.ResolvedSourceConfig{ Git: &corev1alpha1.ResolvedGitSource{ @@ -66,30 +32,32 @@ func (*remoteGitResolver) Resolve(keychain GitKeychain, sourceConfig corev1alpha }, }, nil } + client.InstallProtocol("https", httpsTransport) - // get the remote references - references, err := remote.List(&gogit.ListOptions{ + refs, err := remote.List(&gogit.ListOptions{ Auth: auth, }) if err != nil { - return corev1alpha1.ResolvedSourceConfig{}, errors.Wrap(err, "listing remote references") + return corev1alpha1.ResolvedSourceConfig{ + Git: &corev1alpha1.ResolvedGitSource{ + URL: sourceConfig.Git.URL, + Revision: sourceConfig.Git.Revision, + Type: corev1alpha1.Unknown, + SubPath: sourceConfig.SubPath, + }, + }, nil } - // iterate over the references - for _, reference := range references { - // iterate over the revRefParseRules - for _, revRefParseRule := range refRevParseRules { - // return ResolvedSourceConfig if the sourceConfig.Git.Revision matches the reference.Name() - if fmt.Sprintf(revRefParseRule, sourceConfig.Git.Revision) == reference.Name().String() { - return corev1alpha1.ResolvedSourceConfig{ - Git: &corev1alpha1.ResolvedGitSource{ - URL: sourceConfig.Git.URL, - Revision: reference.Hash().String(), - Type: referenceNameToType(reference.Name()), - SubPath: sourceConfig.SubPath, - }, - }, nil - } + for _, ref := range refs { + if ref.Name().Short() == sourceConfig.Git.Revision { + return corev1alpha1.ResolvedSourceConfig{ + Git: &corev1alpha1.ResolvedGitSource{ + URL: sourceConfig.Git.URL, + Revision: ref.Hash().String(), + Type: sourceType(ref), + SubPath: sourceConfig.SubPath, + }, + }, nil } } @@ -103,38 +71,13 @@ func (*remoteGitResolver) Resolve(keychain GitKeychain, sourceConfig corev1alpha }, nil } -type transportCallbackAdapter struct { - keychain GitKeychain -} - -func keychainAsAuth(keychain GitKeychain, url string, usernameFromUrl string, allowedTypes CredentialType) (transport.AuthMethod, error) { - // Resolve(url string, usernameFromUrl string, allowedTypes CredentialType) (GoGitCredential, error) - goGitCredential, err := keychain.Resolve(url, usernameFromUrl, allowedTypes) - if err != nil { - return nil, err - } - auth, err := goGitCredential.Cred() - if err != nil { - return nil, err - } - return auth, nil -} - -func referenceNameToType(referenceName plumbing.ReferenceName) corev1alpha1.GitSourceKind { +func sourceType(reference *plumbing.Reference) corev1alpha1.GitSourceKind { switch { - case referenceName.IsBranch(): + case reference.Name().IsBranch(): return corev1alpha1.Branch - case referenceName.IsTag(): + case reference.Name().IsTag(): return corev1alpha1.Tag default: return corev1alpha1.Unknown } } - -var refRevParseRules = []string{ - "refs/%s", - "refs/tags/%s", - "refs/heads/%s", - "refs/remotes/%s", - "refs/remotes/%s/HEAD", -} diff --git a/pkg/git/remote_git_resolver_test.go b/pkg/git/remote_git_resolver_test.go index 01af10c28..2b54fe205 100644 --- a/pkg/git/remote_git_resolver_test.go +++ b/pkg/git/remote_git_resolver_test.go @@ -3,6 +3,7 @@ package git import ( "testing" + "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/sclevine/spec" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -26,9 +27,9 @@ func testRemoteGitResolver(t *testing.T, when spec.G, it spec.S) { when("#Resolve", func() { when("source is a commit", func() { it("returns type commit", func() { - gitResolver := &remoteGitResolver{} + gitResolver := remoteGitResolver{} - resolvedGitSource, err := gitResolver.Resolve(&fakeGitKeychain{}, corev1alpha1.SourceConfig{ + resolvedGitSource, err := gitResolver.Resolve(anonymousAuth, corev1alpha1.SourceConfig{ Git: &corev1alpha1.Git{ URL: url, Revision: nonHEADCommit, @@ -37,22 +38,22 @@ func testRemoteGitResolver(t *testing.T, when spec.G, it spec.S) { }) require.NoError(t, err) - assert.Equal(t, resolvedGitSource, corev1alpha1.ResolvedSourceConfig{ + assert.Equal(t, corev1alpha1.ResolvedSourceConfig{ Git: &corev1alpha1.ResolvedGitSource{ URL: url, Revision: nonHEADCommit, SubPath: "/foo/bar", Type: corev1alpha1.Commit, }, - }) + }, resolvedGitSource) }) }) when("source is a branch", func() { it("returns branch with resolved commit", func() { - gitResolver := &remoteGitResolver{} + gitResolver := remoteGitResolver{} - resolvedGitSource, err := gitResolver.Resolve(&fakeGitKeychain{}, corev1alpha1.SourceConfig{ + resolvedGitSource, err := gitResolver.Resolve(anonymousAuth, corev1alpha1.SourceConfig{ Git: &corev1alpha1.Git{ URL: url, Revision: "master", @@ -61,14 +62,14 @@ func testRemoteGitResolver(t *testing.T, when spec.G, it spec.S) { }) require.NoError(t, err) - assert.Equal(t, resolvedGitSource, corev1alpha1.ResolvedSourceConfig{ + assert.Equal(t, corev1alpha1.ResolvedSourceConfig{ Git: &corev1alpha1.ResolvedGitSource{ URL: url, Revision: fixtureHEADMasterCommit, Type: corev1alpha1.Branch, SubPath: "/foo/bar", }, - }) + }, resolvedGitSource) }) }) @@ -76,9 +77,9 @@ func testRemoteGitResolver(t *testing.T, when spec.G, it spec.S) { it("returns tag with resolved commit", func() { tagsUrl := "https://github.com/git-fixtures/tags.git" - gitResolver := &remoteGitResolver{} + gitResolver := remoteGitResolver{} - resolvedGitSource, err := gitResolver.Resolve(&fakeGitKeychain{}, corev1alpha1.SourceConfig{ + resolvedGitSource, err := gitResolver.Resolve(anonymousAuth, corev1alpha1.SourceConfig{ Git: &corev1alpha1.Git{ URL: tagsUrl, Revision: tag, @@ -87,38 +88,40 @@ func testRemoteGitResolver(t *testing.T, when spec.G, it spec.S) { }) require.NoError(t, err) - assert.Equal(t, resolvedGitSource, corev1alpha1.ResolvedSourceConfig{ + assert.Equal(t, corev1alpha1.ResolvedSourceConfig{ Git: &corev1alpha1.ResolvedGitSource{ URL: tagsUrl, Revision: tagCommit, Type: corev1alpha1.Tag, SubPath: "/foo/bar", }, - }) + }, resolvedGitSource) }) }) when("authentication fails", func() { it("returns an unknown type", func() { - gitResolver := &remoteGitResolver{} + gitResolver := remoteGitResolver{} - resolvedGitSource, err := gitResolver.Resolve(&fakeGitKeychain{}, corev1alpha1.SourceConfig{ + resolvedGitSource, _ := gitResolver.Resolve(&http.BasicAuth{ + Username: "bad-username", + Password: "bad-password", + }, corev1alpha1.SourceConfig{ Git: &corev1alpha1.Git{ URL: "git@localhost:org/repo", Revision: tag, }, SubPath: "/foo/bar", }) - require.NoError(t, err) - assert.Equal(t, resolvedGitSource, corev1alpha1.ResolvedSourceConfig{ + assert.Equal(t, corev1alpha1.ResolvedSourceConfig{ Git: &corev1alpha1.ResolvedGitSource{ URL: "git@localhost:org/repo", Revision: tag, Type: corev1alpha1.Unknown, SubPath: "/foo/bar", }, - }) + }, resolvedGitSource) }) }) }) diff --git a/pkg/git/resolver.go b/pkg/git/resolver.go index 948ffccbb..0dc5237b9 100644 --- a/pkg/git/resolver.go +++ b/pkg/git/resolver.go @@ -11,23 +11,23 @@ import ( type Resolver struct { remoteGitResolver remoteGitResolver - gitKeychain *k8sGitKeychainFactory + gitKeychain *k8sGitKeychain } func NewResolver(k8sClient k8sclient.Interface) *Resolver { return &Resolver{ remoteGitResolver: remoteGitResolver{}, - gitKeychain: newK8sGitKeychainFactory(k8sClient), + gitKeychain: newK8sGitKeychain(k8sClient), } } func (r *Resolver) Resolve(ctx context.Context, sourceResolver *buildapi.SourceResolver) (corev1alpha1.ResolvedSourceConfig, error) { - keychain, err := r.gitKeychain.KeychainForServiceAccount(ctx, sourceResolver.Namespace, sourceResolver.Spec.ServiceAccountName) + auth, err := r.gitKeychain.Resolve(ctx, sourceResolver.Namespace, sourceResolver.Spec.ServiceAccountName, *sourceResolver.Spec.Source.Git) if err != nil { return corev1alpha1.ResolvedSourceConfig{}, err } - return r.remoteGitResolver.Resolve(keychain, sourceResolver.Spec.Source) + return r.remoteGitResolver.Resolve(auth, sourceResolver.Spec.Source) } func (*Resolver) CanResolve(sourceResolver *buildapi.SourceResolver) bool { diff --git a/pkg/git/url_parser.go b/pkg/git/url_parser.go index 9e5f70dc3..e0ad0e900 100644 --- a/pkg/git/url_parser.go +++ b/pkg/git/url_parser.go @@ -8,7 +8,7 @@ import ( var shortScpRegex = regexp.MustCompile(`^(ssh://)?(.*)@([[:alnum:]\.-]+):(.*)$`) // ParseURL converts a short scp-like SSH syntax to a proper SSH URL. -// Git's ssh protocol supports a url like user@hostname:path syntax, which is +// Git's ssh protocol supports a URL like user@hostname:path syntax, which is // not a valid ssh url but is inherited from scp. Because the library we // use for git relies on the Golang SSH support, we need to convert it to a // proper SSH URL.