Do not refresh if no repo changes (#4085)

* Do not refresh if no repo changes Signed-off-by: John Belamaric <jbelamaric@google.com> * Need to hold the lock during Version Signed-off-by: John Belamaric <jbelamaric@google.com> * Fix fake repo Signed-off-by: John Belamaric <jbelamaric@google.com> --------- Signed-off-by: John Belamaric <jbelamaric@google.com>
kptdev · Nov 17, 2023 · 5a80478 · 5a80478
1 parent 7716dd3
commit 5a80478
Show file tree

Hide file tree

Showing 5 changed files with 116 additions and 0 deletions.
diff --git a/porch/pkg/cache/repository.go b/porch/pkg/cache/repository.go
@@ -52,6 +52,8 @@ type cachedRepository struct {
 	repo     repository.Repository
 	cancel   context.CancelFunc
 
+	lastVersion string
+
 	mutex                  sync.Mutex
 	cachedPackageRevisions map[repository.PackageRevisionKey]*cachedPackageRevision
 	cachedPackages         map[repository.PackageKey]*cachedPackage
@@ -86,6 +88,10 @@ func newRepository(id string, repoSpec *configapi.Repository, repo repository.Re
 	return r
 }
 
+func (r *cachedRepository) Version(ctx context.Context) (string, error) {
+	return r.repo.Version(ctx)
+}
+
 func (r *cachedRepository) ListPackageRevisions(ctx context.Context, filter repository.ListPackageRevisionFilter) ([]repository.PackageRevision, error) {
 	packages, err := r.getPackageRevisions(ctx, filter, false)
 	if err != nil {
@@ -390,6 +396,15 @@ func (r *cachedRepository) refreshAllCachedPackages(ctx context.Context) (map[re
 	start := time.Now()
 	defer func() { klog.Infof("repo %s: refresh finished in %f secs", r.id, time.Since(start).Seconds()) }()
 
+	curVer, err := r.Version(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if curVer == r.lastVersion {
+		return r.cachedPackages, r.cachedPackageRevisions, nil
+	}
+
 	// Look up all existing PackageRevCRs so we an compare those to the
 	// actual Packagerevisions found in git/oci, and add/prune PackageRevCRs
 	// as necessary.
@@ -538,6 +553,7 @@ func (r *cachedRepository) refreshAllCachedPackages(ctx context.Context) (map[re
 
 	r.cachedPackageRevisions = newPackageRevisionMap
 	r.cachedPackages = newPackageMap
+	r.lastVersion = curVer
 
 	return newPackageMap, newPackageRevisionMap, nil
 }
diff --git a/porch/pkg/engine/fake/repository.go b/porch/pkg/engine/fake/repository.go
@@ -34,6 +34,10 @@ func (r *Repository) Close() error {
 	return nil
 }
 
+func (r *Repository) Version(ctx context.Context) (string, error) {
+	return "foo", nil
+}
+
 func (r *Repository) ListPackageRevisions(_ context.Context, filter repository.ListPackageRevisionFilter) ([]repository.PackageRevision, error) {
 	var revs []repository.PackageRevision
 	for _, rev := range r.PackageRevisions {

diff --git a/porch/pkg/git/git.go b/porch/pkg/git/git.go
@@ -15,7 +15,10 @@
 package git
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
@@ -186,6 +189,35 @@ func (r *gitRepository) Close() error {
 	return nil
 }
 
+func (r *gitRepository) Version(ctx context.Context) (string, error) {
+	ctx, span := tracer.Start(ctx, "gitRepository::Version", trace.WithAttributes())
+	defer span.End()
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	if err := r.fetchRemoteRepository(ctx); err != nil {
+		return "", err
+	}
+
+	refs, err := r.repo.References()
+	if err != nil {
+		return "", err
+	}
+
+	b := bytes.Buffer{}
+	for {
+		ref, err := refs.Next()
+		if err == io.EOF {
+			break
+		}
+
+		b.WriteString(ref.String())
+	}
+
+	hash := sha256.Sum256(b.Bytes())
+	return hex.EncodeToString(hash[:]), nil
+}
+
 func (r *gitRepository) ListPackages(ctx context.Context, filter repository.ListPackageFilter) ([]repository.Package, error) {
 	ctx, span := tracer.Start(ctx, "gitRepository::ListPackages", trace.WithAttributes())
 	defer span.End()
@@ -629,6 +661,9 @@ func (r *gitRepository) discoverFinalizedPackages(ctx context.Context, ref *plum
 
 // loadDraft will load the draft package.  If the package isn't found (we now require a Kptfile), it will return (nil, nil)
 func (r *gitRepository) loadDraft(ctx context.Context, ref *plumbing.Reference) (*gitPackageRevision, error) {
+	ctx, span := tracer.Start(ctx, "gitRepository::loadDraft", trace.WithAttributes())
+	defer span.End()
+
 	name, workspaceName, err := parseDraftName(ref)
 	if err != nil {
 		return nil, err
@@ -719,6 +754,9 @@ func parseDraftName(draft *plumbing.Reference) (name string, workspaceName v1alp
 }
 
 func (r *gitRepository) loadTaggedPackages(ctx context.Context, tag *plumbing.Reference) ([]*gitPackageRevision, error) {
+	ctx, span := tracer.Start(ctx, "gitRepository::loadTaggedPackages", trace.WithAttributes())
+	defer span.End()
+
 	name, ok := getTagNameInLocalRepo(tag.Name())
 	if !ok {
 		return nil, fmt.Errorf("invalid tag ref: %q", tag)

diff --git a/porch/pkg/oci/oci.go b/porch/pkg/oci/oci.go
@@ -15,8 +15,10 @@
 package oci
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha1"
+	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
 	"strings"
@@ -67,6 +69,59 @@ func (r *ociRepository) Close() error {
 	return nil
 }
 
+// there is probably a more efficient way to do this
+func (r *ociRepository) Version(ctx context.Context) (string, error) {
+	ctx, span := tracer.Start(ctx, "ociRepository::Version")
+	defer span.End()
+
+	if r.content != configapi.RepositoryContentPackage {
+		return "", nil
+	}
+
+	ociRepo, err := name.NewRepository(r.spec.Registry)
+	if err != nil {
+		return "", err
+	}
+
+	options := r.storage.CreateOptions(ctx)
+
+	tags, err := google.List(ociRepo, options...)
+	if err != nil {
+		return "", err
+	}
+
+	klog.Infof("tags: %#v", tags)
+
+	b := bytes.Buffer{}
+	for _, childName := range tags.Children {
+		path := fmt.Sprintf("%s/%s", r.spec.Registry, childName)
+		child, err := name.NewRepository(path, name.StrictValidation)
+		if err != nil {
+			klog.Warningf("Cannot create nested repository %q: %v", path, err)
+			continue
+		}
+
+		childTags, err := google.List(child, options...)
+		if err != nil {
+			klog.Warningf("Cannot list nested repository %q: %v", path, err)
+			continue
+		}
+
+		// klog.Infof("childTags: %#v", childTags)
+
+		for digest, m := range childTags.Manifests {
+			b.WriteString(digest)
+			mb, err := m.MarshalJSON()
+			if err != nil {
+				return "", err
+			}
+			b.Write(mb)
+		}
+	}
+	hash := sha256.Sum256(b.Bytes())
+	return hex.EncodeToString(hash[:]), nil
+}
+
 func (r *ociRepository) ListPackageRevisions(ctx context.Context, filter repository.ListPackageRevisionFilter) ([]repository.PackageRevision, error) {
 	if r.content != configapi.RepositoryContentPackage {
 		return []repository.PackageRevision{}, nil

diff --git a/porch/pkg/repository/repository.go b/porch/pkg/repository/repository.go
@@ -206,6 +206,9 @@ type Repository interface {
 	// DeletePackage deletes a package
 	DeletePackage(ctx context.Context, old Package) error
 
+	// Version returns a string that is guaranteed to be different if any change has been made to the repo contents
+	Version(ctx context.Context) (string, error)
+
 	// Close cleans up any resources associated with the repository
 	Close() error
 }