Skip to content

Commit

Permalink
Refactor hard prune
Browse files Browse the repository at this point in the history 
Signed-off-by: Gladkov Alexey <agladkov@redhat.com>
  • Loading branch information
@legionus
legionus committed Oct 25, 2017
1 parent 337ee95 commit eed4633
Show file tree
Hide file tree
Showing 2 changed files with 132 additions and 39 deletions.
10 changes: 9 additions & 1 deletion pkg/cmd/dockerregistry/dockerregistry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,15 @@ func ExecutePruner(configFile io.Reader, dryRun bool) {
log.Fatalf("error creating registry: %s", err)
}

stats, err := prune.Prune(ctx, storageDriver, registry, registryClient, dryRun)
var pruner prune.Pruner

if dryRun {
pruner = &prune.DryRunPruner{}
} else {
pruner = &prune.RegistryPruner{storageDriver}
}

stats, err := prune.Prune(ctx, registry, registryClient, pruner)
if err != nil {
log.Error(err)
}
Expand Down
161 changes: 123 additions & 38 deletions pkg/dockerregistry/server/prune/prune.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,112 @@ import (
imageapiv1 "github.com/openshift/origin/pkg/image/apis/image/v1"
)

// Pruner defines a common set of operations for pruning
type Pruner interface {
DeleteRepository(ctx context.Context, reponame string) error
DeleteManifestLink(ctx context.Context, svc distribution.ManifestService, reponame string, dgst digest.Digest) error
DeleteBlob(ctx context.Context, dgst digest.Digest) error
}

// DryRunPruner prints information about each object that going to remove.
type DryRunPruner struct{}

var _ Pruner = &DryRunPruner{}

func (p *DryRunPruner) DeleteRepository(ctx context.Context, reponame string) error {
logger := context.GetLogger(ctx)
logger.Printf("Would delete repository: %s", reponame)
return nil
}

func (p *DryRunPruner) DeleteManifestLink(ctx context.Context, svc distribution.ManifestService, reponame string, dgst digest.Digest) error {
logger := context.GetLogger(ctx)
logger.Printf("Would delete manifest link: %s@%s", reponame, dgst)
return nil
}

func (p *DryRunPruner) DeleteBlob(ctx context.Context, dgst digest.Digest) error {
logger := context.GetLogger(ctx)
logger.Printf("Would delete blob: %s", dgst)
return nil
}

// RegistryPruner deletes objects.
type RegistryPruner struct {
StorageDriver driver.StorageDriver
}

var _ Pruner = &RegistryPruner{}

func (p *RegistryPruner) DeleteRepository(ctx context.Context, reponame string) error {
logger := context.GetLogger(ctx)
vacuum := storage.NewVacuum(ctx, p.StorageDriver)

logger.Debugln("Removing %s repository", reponame)
if err := vacuum.RemoveRepository(reponame); err != nil {
return fmt.Errorf("unable to remove the repository %s: %v", reponame, err)
}

return nil
}

func (p *RegistryPruner) DeleteManifestLink(ctx context.Context, svc distribution.ManifestService, reponame string, dgst digest.Digest) error {
logger := context.GetLogger(ctx)

logger.Printf("Deleting manifest link: %s@%s", reponame, dgst)
if err := svc.Delete(ctx, dgst); err != nil {
return fmt.Errorf("failed to delete the manifest link %s@%s: %v", reponame, dgst, err)
}

return nil
}

func (p *RegistryPruner) DeleteBlob(ctx context.Context, dgst digest.Digest) error {
vacuum := storage.NewVacuum(ctx, p.StorageDriver)

if err := vacuum.RemoveBlob(string(dgst)); err != nil {
return fmt.Errorf("failed to delete the blob %s: %v", dgst, err)
}

return nil
}

type garbageColletor struct {
Pruner Pruner

repoName string

manifestService distribution.ManifestService
manifestRepo string
manifestLink digest.Digest
}

func (gc *garbageColletor) SetRepository(repoName string) {
gc.repoName = repoName
}

func (gc *garbageColletor) SetManifestLink(svc distribution.ManifestService, repoName string, dgst digest.Digest) {
gc.manifestService = svc
gc.manifestRepo = repoName
gc.manifestLink = dgst
}

func (gc *garbageColletor) Collect(ctx context.Context) error {
if len(gc.manifestLink) > 0 {
if err := gc.Pruner.DeleteManifestLink(ctx, gc.manifestService, gc.manifestRepo, gc.manifestLink); err != nil {
return err
}
gc.manifestLink = ""
}
if len(gc.repoName) > 0 {
if err := gc.Pruner.DeleteRepository(ctx, gc.repoName); err != nil {
return err
}
gc.repoName = ""
}
return nil
}

func imageStreamHasManifestDigest(is *imageapiv1.ImageStream, dgst digest.Digest) bool {
for _, tagEventList := range is.Status.Tags {
for _, tagEvent := range tagEventList.Items {
Expand All @@ -42,7 +148,7 @@ type Summary struct {
//
// TODO(dmage): remove layer links to a blob if the blob is removed or it doesn't belong to the ImageStream.
// TODO(dmage): keep young blobs (docker/distribution#2297).
func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, registryClient client.RegistryClient, dryRun bool) (Summary, error) {
func Prune(ctx context.Context, registry distribution.Namespace, registryClient client.RegistryClient, pruner Pruner) (Summary, error) {
logger := context.GetLogger(ctx)

repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
Expand Down Expand Up @@ -84,8 +190,14 @@ func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry dis

var stats Summary

var reposToDelete []string
gc := &garbageColletor{
Pruner: pruner,
}

err = repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
if err := gc.Collect(ctx); err != nil {
return err
}
logger.Debugln("Processing repository", repoName)

named, err := reference.WithName(repoName)
Expand All @@ -101,10 +213,7 @@ func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry dis
is, err := oc.ImageStreams(ref.Namespace).Get(ref.Name, metav1.GetOptions{})
if kerrors.IsNotFound(err) {
logger.Printf("The image stream %s/%s is not found, will remove the whole repository", ref.Namespace, ref.Name)

// We cannot delete the repository at this point, because it would break Enumerate.
reposToDelete = append(reposToDelete, repoName)

gc.SetRepository(repoName)
return nil
} else if err != nil {
return fmt.Errorf("failed to get the image stream %s: %v", repoName, err)
Expand All @@ -126,21 +235,16 @@ func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry dis
}

err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
if _, ok := inuse[string(dgst)]; ok && imageStreamHasManifestDigest(is, dgst) {
logger.Debugf("Keeping the manifest link %s@%s", repoName, dgst)
return nil
if err := gc.Collect(ctx); err != nil {
return err
}

if dryRun {
logger.Printf("Would delete manifest link: %s@%s", repoName, dgst)
if _, ok := inuse[string(dgst)]; ok && imageStreamHasManifestDigest(is, dgst) {
logger.Debugf("Keeping the manifest link %s@%s", repoName, dgst)
return nil
}

logger.Printf("Deleting manifest link: %s@%s", repoName, dgst)
if err := manifestService.Delete(ctx, dgst); err != nil {
return fmt.Errorf("failed to delete the manifest link %s@%s: %v", repoName, dgst, err)
}

gc.SetManifestLink(manifestService, repoName, dgst)
return nil
})
if e, ok := err.(driver.PathNotFoundError); ok {
Expand All @@ -158,18 +262,8 @@ func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry dis
return stats, err
}

vacuum := storage.NewVacuum(ctx, storageDriver)

logger.Debugln("Removing repositories")
for _, repoName := range reposToDelete {
if dryRun {
logger.Printf("Would delete repository: %s", repoName)
continue
}

if err = vacuum.RemoveRepository(repoName); err != nil {
return stats, fmt.Errorf("unable to remove the repository %s: %v", repoName, err)
}
if err := gc.Collect(ctx); err != nil {
return stats, err
}

logger.Debugln("Processing blobs")
Expand All @@ -188,16 +282,7 @@ func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry dis
stats.Blobs++
stats.DiskSpace += desc.Size

if dryRun {
logger.Printf("Would delete blob: %s", dgst)
return nil
}

if err := vacuum.RemoveBlob(string(dgst)); err != nil {
return fmt.Errorf("failed to delete the blob %s: %v", dgst, err)
}

return nil
return pruner.DeleteBlob(ctx, dgst)
})
return stats, err
}

0 comments on commit eed4633

Please sign in to comment.