Add Clone Package Test (#2807)

* Add Clone Package Test
* Fix issues with git server's reporting of symbolic references

porch/engine/go.mod

@@ -9,6 +9,8 @@ require (
 	github.com/GoogleContainerTools/kpt/porch/controllers v0.0.0-00010101000000-000000000000
 	github.com/GoogleContainerTools/kpt/porch/func v0.0.0-00010101000000-000000000000
 	github.com/GoogleContainerTools/kpt/porch/repository v0.0.0-00010101000000-000000000000
+	github.com/go-git/go-billy/v5 v5.3.1
+	github.com/go-git/go-git/v5 v5.4.3-0.20220119145113-935af59cf64f
 	github.com/google/go-cmp v0.5.7
 	google.golang.org/grpc v1.44.0
 	k8s.io/klog/v2 v2.40.1
@@ -35,8 +37,6 @@ require (
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
-	github.com/go-git/go-billy/v5 v5.3.1 // indirect
-	github.com/go-git/go-git/v5 v5.4.3-0.20220119145113-935af59cf64f // indirect
 	github.com/go-logr/logr v1.2.1 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.5 // indirect

porch/engine/pkg/engine/clone_test.go

@@ -0,0 +1,185 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package engine
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/GoogleContainerTools/kpt/porch/api/porch/v1alpha1"
+	"github.com/GoogleContainerTools/kpt/porch/repository/pkg/git"
+	"github.com/GoogleContainerTools/kpt/porch/repository/pkg/repository"
+	"github.com/go-git/go-billy/v5/memfs"
+	gogit "github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/storage/memory"
+)
+
+func createRepoWithContents(t *testing.T, contentDir string) *gogit.Repository {
+	repo, err := gogit.Init(memory.NewStorage(), memfs.New())
+	if err != nil {
+		t.Fatalf("Failed to initialize in-memory git repository: %v", err)
+	}
+	wt, err := repo.Worktree()
+	if err != nil {
+		t.Fatalf("Failed to get git repository worktree: %v", err)
+	}
+
+	if err := filepath.Walk(contentDir, func(path string, info fs.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		} else if !info.Mode().IsRegular() {
+			return fmt.Errorf("irredular file object detected: %q (%s)", path, info.Mode())
+		}
+		rel, err := filepath.Rel(contentDir, path)
+		if err != nil {
+			return fmt.Errorf("failed to get relative path from %q to %q: %w", contentDir, path, err)
+		}
+		dir := filepath.Dir(rel)
+		if err := wt.Filesystem.MkdirAll(dir, 0777); err != nil {
+			return fmt.Errorf("failed to create directories for %q: %w", rel, err)
+		}
+		src, err := os.Open(path)
+		if err != nil {
+			return fmt.Errorf("failed to open the source file %q: %w", path, err)
+		}
+		defer src.Close()
+		dst, err := wt.Filesystem.Create(rel)
+		if err != nil {
+			return fmt.Errorf("failed to create the destination file %q: %w", rel, err)
+		}
+		defer dst.Close()
+		if _, err := io.Copy(dst, src); err != nil {
+			return fmt.Errorf("failed to copy file contents %q -> %q: %w", path, rel, err)
+		}
+		wt.Add(rel)
+		return nil
+	}); err != nil {
+		t.Fatalf("Failed populating Git repository worktree: %v", err)
+	}
+
+	sig := object.Signature{
+		Name:  "Porch Unit Test",
+		Email: "porch-unit-test@kpt.dev",
+		When:  time.Now(),
+	}
+
+	hash, err := wt.Commit("Initial Commit", &gogit.CommitOptions{
+		All:       true,
+		Author:    &sig,
+		Committer: &sig,
+	})
+	if err != nil {
+		t.Fatalf("Failed creating initial commit: %v", err)
+	}
+
+	main := plumbing.NewHashReference(plumbing.ReferenceName("refs/heads/main"), hash)
+	if err := repo.Storer.SetReference(main); err != nil {
+		t.Fatalf("Failed to set refs/heads/main to commit sha %s", hash)
+	}
+	head := plumbing.NewSymbolicReference(plumbing.HEAD, "refs/heads/main")
+	if err := repo.Storer.SetReference(head); err != nil {
+		t.Fatalf("Failed to set HEAD to refs/heads/main: %v", err)
+	}
+
+	_ = repo.Storer.RemoveReference(plumbing.Master)
+
+	return repo
+}
+
+func startGitServer(t *testing.T, repo *gogit.Repository) string {
+	server, err := git.NewGitServer(repo)
+	if err != nil {
+		t.Fatalf("Failed to create git server: %v", err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	addressChannel := make(chan net.Addr)
+
+	go func() {
+		err := server.ListenAndServe(ctx, "127.0.0.1:0", addressChannel)
+		if err != nil {
+			if err == http.ErrServerClosed {
+				t.Log("Git server shut down successfully")
+			} else {
+				t.Errorf("Git server exited with error: %v", err)
+			}
+		}
+	}()
+
+	// Wait for server to start up
+	address, ok := <-addressChannel
+	if !ok {
+		t.Fatalf("Server failed to start")
+		return ""
+	}
+
+	return fmt.Sprintf("http://%s", address)
+}
+
+type cr struct{}
+
+func (cr) ResolveCredential(ctx context.Context, namespace, name string) (repository.Credential, error) {
+	return repository.Credential{
+		Data: map[string][]byte{
+			"username": []byte(""),
+			"password": []byte(""),
+		},
+	}, nil
+}
+
+func TestCloneGitBasicAuth(t *testing.T) {
+	testdata, err := filepath.Abs(filepath.Join(".", "testdata", "clone"))
+	if err != nil {
+		t.Fatalf("Failed to find testdata: %v", err)
+	}
+	repo := createRepoWithContents(t, testdata)
+	addr := startGitServer(t, repo)
+
+	cpm := clonePackageMutation{
+		task: &v1alpha1.Task{
+			Type: "clone",
+			Clone: &v1alpha1.PackageCloneTaskSpec{
+				Upstream: v1alpha1.UpstreamPackage{
+					Type: "git",
+					Git: &v1alpha1.GitPackage{
+						Repo:      addr,
+						Ref:       "main",
+						Directory: "configmap",
+					},
+				},
+			},
+		},
+		name: "test-configmap",
+	}
+
+	r, _, err := cpm.Apply(context.Background(), repository.PackageResources{})
+	if err != nil {
+		t.Errorf("task apply failed: %v", err)
+	}
+
+	t.Logf("%v", r)
+}

porch/engine/pkg/engine/testdata/clone/bucket/Kptfile

@@ -0,0 +1,20 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kpt.dev/v1
+kind: Kptfile
+metadata:
+  name: bucket
+info:
+  description: Bucket test package

porch/engine/pkg/engine/testdata/clone/bucket/bucket.yaml

@@ -0,0 +1,26 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: storage.cnrm.cloud.google.com/v1beta1
+kind: StorageBucket
+metadata:
+  name: bucket-name
+  namespace: bucket-namespace
+  annotations:
+    cnrm.cloud.google.com/project-id: bucket-project
+spec:
+  storageClass: standard
+  uniformBucketLevelAccess: true
+  versioning:
+    enabled: false

porch/engine/pkg/engine/testdata/clone/configmap/Kptfile

@@ -0,0 +1,20 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kpt.dev/v1
+kind: Kptfile
+metadata:
+  name: configmap
+info:
+  description: ConfigMap test package

porch/engine/pkg/engine/testdata/clone/configmap/configmap.yaml

@@ -0,0 +1,21 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: configmap-name
+  namespace: configmap-namespace
+data:
+  key: value

porch/repository/pkg/git/testing.go

@@ -37,6 +37,10 @@ import (
 	"k8s.io/klog/v2"
 )
 
+const (
+	Main plumbing.ReferenceName = "refs/heads/main"
+)
+
 // GitServer is a mock git server implementing "just enough" of the git protocol
 type GitServer struct {
 	repo *gogit.Repository
@@ -143,8 +147,30 @@ func (s *GitServer) serveGitInfoRefs(w http.ResponseWriter, r *http.Request) err
 			klog.Infof("skipping remote ref %q", name)
 			return nil
 		}
-		s := fmt.Sprintf("%s %s", ref.Hash().String(), name)
-		refs = append(refs, s)
+
+		var resolved *plumbing.Reference
+		switch ref.Type() {
+		case plumbing.SymbolicReference:
+			if r, err := s.repo.Reference(ref.Name(), true); err != nil {
+				klog.Warningf("Skippling unresolvable symbolic reference %q: %w", ref.Name(), err)
+				return nil
+			} else {
+				resolved = r
+			}
+		case plumbing.HashReference:
+			resolved = ref
+		default:
+			return fmt.Errorf("unexpected reference encountered: %s", ref)
+		}
+
+		s := fmt.Sprintf("%s %s", resolved.Hash().String(), name)
+
+		// https://git-scm.com/docs/http-protocol: HEAD SHOULD be first
+		if name == plumbing.HEAD {
+			refs = append([]string{s}, refs...)
+		} else {
+			refs = append(refs, s)
+		}
 		return nil
 	}); err != nil {
 		return fmt.Errorf("error iterating through references: %w", err)
@@ -550,9 +576,18 @@ func initRepo(repo *git.Repository) error {
 	}
 
 	{
-		ref := plumbing.NewHashReference("refs/heads/main", commitHash)
+		ref := plumbing.NewHashReference(Main, commitHash)
 		if err := repo.Storer.SetReference(ref); err != nil {
-			return fmt.Errorf("error setting reference: %w", err)
+			return fmt.Errorf("error setting reference %q: %w", Main, err)
+		}
+
+		// gogit uses suboptimal default reference name; delete it
+		repo.Storer.RemoveReference(plumbing.Master)
+
+		// create correct HEAD as a symbolic reference of main branch
+		head := plumbing.NewSymbolicReference(plumbing.HEAD, Main)
+		if err := repo.Storer.SetReference(head); err != nil {
+			return fmt.Errorf("error creating HEAD ref: %w", err)
 		}
 	}