From 5a804788c334da4c9ed09ce13aa48231a39a917a Mon Sep 17 00:00:00 2001 From: John Belamaric Date: Fri, 17 Nov 2023 09:18:53 -0800 Subject: [PATCH] Do not refresh if no repo changes (#4085) * Do not refresh if no repo changes Signed-off-by: John Belamaric * Need to hold the lock during Version Signed-off-by: John Belamaric * Fix fake repo Signed-off-by: John Belamaric --------- Signed-off-by: John Belamaric --- porch/pkg/cache/repository.go | 16 +++++++++ porch/pkg/engine/fake/repository.go | 4 +++ porch/pkg/git/git.go | 38 ++++++++++++++++++++ porch/pkg/oci/oci.go | 55 +++++++++++++++++++++++++++++ porch/pkg/repository/repository.go | 3 ++ 5 files changed, 116 insertions(+) diff --git a/porch/pkg/cache/repository.go b/porch/pkg/cache/repository.go index f9863ab93..3c95b6c96 100644 --- a/porch/pkg/cache/repository.go +++ b/porch/pkg/cache/repository.go @@ -52,6 +52,8 @@ type cachedRepository struct { repo repository.Repository cancel context.CancelFunc + lastVersion string + mutex sync.Mutex cachedPackageRevisions map[repository.PackageRevisionKey]*cachedPackageRevision cachedPackages map[repository.PackageKey]*cachedPackage @@ -86,6 +88,10 @@ func newRepository(id string, repoSpec *configapi.Repository, repo repository.Re return r } +func (r *cachedRepository) Version(ctx context.Context) (string, error) { + return r.repo.Version(ctx) +} + func (r *cachedRepository) ListPackageRevisions(ctx context.Context, filter repository.ListPackageRevisionFilter) ([]repository.PackageRevision, error) { packages, err := r.getPackageRevisions(ctx, filter, false) if err != nil { @@ -390,6 +396,15 @@ func (r *cachedRepository) refreshAllCachedPackages(ctx context.Context) (map[re start := time.Now() defer func() { klog.Infof("repo %s: refresh finished in %f secs", r.id, time.Since(start).Seconds()) }() + curVer, err := r.Version(ctx) + if err != nil { + return nil, nil, err + } + + if curVer == r.lastVersion { + return r.cachedPackages, r.cachedPackageRevisions, nil + } + // Look up all existing PackageRevCRs so we an compare those to the // actual Packagerevisions found in git/oci, and add/prune PackageRevCRs // as necessary. @@ -538,6 +553,7 @@ func (r *cachedRepository) refreshAllCachedPackages(ctx context.Context) (map[re r.cachedPackageRevisions = newPackageRevisionMap r.cachedPackages = newPackageMap + r.lastVersion = curVer return newPackageMap, newPackageRevisionMap, nil } diff --git a/porch/pkg/engine/fake/repository.go b/porch/pkg/engine/fake/repository.go index e94969319..cd2ba340a 100644 --- a/porch/pkg/engine/fake/repository.go +++ b/porch/pkg/engine/fake/repository.go @@ -34,6 +34,10 @@ func (r *Repository) Close() error { return nil } +func (r *Repository) Version(ctx context.Context) (string, error) { + return "foo", nil +} + func (r *Repository) ListPackageRevisions(_ context.Context, filter repository.ListPackageRevisionFilter) ([]repository.PackageRevision, error) { var revs []repository.PackageRevision for _, rev := range r.PackageRevisions { diff --git a/porch/pkg/git/git.go b/porch/pkg/git/git.go index a4b08caf0..bbf1f8450 100644 --- a/porch/pkg/git/git.go +++ b/porch/pkg/git/git.go @@ -15,7 +15,10 @@ package git import ( + "bytes" "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "io" @@ -186,6 +189,35 @@ func (r *gitRepository) Close() error { return nil } +func (r *gitRepository) Version(ctx context.Context) (string, error) { + ctx, span := tracer.Start(ctx, "gitRepository::Version", trace.WithAttributes()) + defer span.End() + r.mutex.Lock() + defer r.mutex.Unlock() + + if err := r.fetchRemoteRepository(ctx); err != nil { + return "", err + } + + refs, err := r.repo.References() + if err != nil { + return "", err + } + + b := bytes.Buffer{} + for { + ref, err := refs.Next() + if err == io.EOF { + break + } + + b.WriteString(ref.String()) + } + + hash := sha256.Sum256(b.Bytes()) + return hex.EncodeToString(hash[:]), nil +} + func (r *gitRepository) ListPackages(ctx context.Context, filter repository.ListPackageFilter) ([]repository.Package, error) { ctx, span := tracer.Start(ctx, "gitRepository::ListPackages", trace.WithAttributes()) defer span.End() @@ -629,6 +661,9 @@ func (r *gitRepository) discoverFinalizedPackages(ctx context.Context, ref *plum // loadDraft will load the draft package. If the package isn't found (we now require a Kptfile), it will return (nil, nil) func (r *gitRepository) loadDraft(ctx context.Context, ref *plumbing.Reference) (*gitPackageRevision, error) { + ctx, span := tracer.Start(ctx, "gitRepository::loadDraft", trace.WithAttributes()) + defer span.End() + name, workspaceName, err := parseDraftName(ref) if err != nil { return nil, err @@ -719,6 +754,9 @@ func parseDraftName(draft *plumbing.Reference) (name string, workspaceName v1alp } func (r *gitRepository) loadTaggedPackages(ctx context.Context, tag *plumbing.Reference) ([]*gitPackageRevision, error) { + ctx, span := tracer.Start(ctx, "gitRepository::loadTaggedPackages", trace.WithAttributes()) + defer span.End() + name, ok := getTagNameInLocalRepo(tag.Name()) if !ok { return nil, fmt.Errorf("invalid tag ref: %q", tag) diff --git a/porch/pkg/oci/oci.go b/porch/pkg/oci/oci.go index 306a27aca..e58a5e685 100644 --- a/porch/pkg/oci/oci.go +++ b/porch/pkg/oci/oci.go @@ -15,8 +15,10 @@ package oci import ( + "bytes" "context" "crypto/sha1" + "crypto/sha256" "encoding/hex" "fmt" "strings" @@ -67,6 +69,59 @@ func (r *ociRepository) Close() error { return nil } +// there is probably a more efficient way to do this +func (r *ociRepository) Version(ctx context.Context) (string, error) { + ctx, span := tracer.Start(ctx, "ociRepository::Version") + defer span.End() + + if r.content != configapi.RepositoryContentPackage { + return "", nil + } + + ociRepo, err := name.NewRepository(r.spec.Registry) + if err != nil { + return "", err + } + + options := r.storage.CreateOptions(ctx) + + tags, err := google.List(ociRepo, options...) + if err != nil { + return "", err + } + + klog.Infof("tags: %#v", tags) + + b := bytes.Buffer{} + for _, childName := range tags.Children { + path := fmt.Sprintf("%s/%s", r.spec.Registry, childName) + child, err := name.NewRepository(path, name.StrictValidation) + if err != nil { + klog.Warningf("Cannot create nested repository %q: %v", path, err) + continue + } + + childTags, err := google.List(child, options...) + if err != nil { + klog.Warningf("Cannot list nested repository %q: %v", path, err) + continue + } + + // klog.Infof("childTags: %#v", childTags) + + for digest, m := range childTags.Manifests { + b.WriteString(digest) + mb, err := m.MarshalJSON() + if err != nil { + return "", err + } + b.Write(mb) + } + } + hash := sha256.Sum256(b.Bytes()) + return hex.EncodeToString(hash[:]), nil +} + func (r *ociRepository) ListPackageRevisions(ctx context.Context, filter repository.ListPackageRevisionFilter) ([]repository.PackageRevision, error) { if r.content != configapi.RepositoryContentPackage { return []repository.PackageRevision{}, nil diff --git a/porch/pkg/repository/repository.go b/porch/pkg/repository/repository.go index 2c4f150d7..9b8a89a2b 100644 --- a/porch/pkg/repository/repository.go +++ b/porch/pkg/repository/repository.go @@ -206,6 +206,9 @@ type Repository interface { // DeletePackage deletes a package DeletePackage(ctx context.Context, old Package) error + // Version returns a string that is guaranteed to be different if any change has been made to the repo contents + Version(ctx context.Context) (string, error) + // Close cleans up any resources associated with the repository Close() error }