Skip to content

Commit

Permalink
Refactor hard prune
Browse files Browse the repository at this point in the history 
Signed-off-by: Gladkov Alexey <agladkov@redhat.com>
  • Loading branch information
@legionus
legionus committed Oct 25, 2017
1 parent 337ee95 commit 79afcb3
Show file tree
Hide file tree
Showing 2 changed files with 151 additions and 41 deletions.
10 changes: 9 additions & 1 deletion pkg/cmd/dockerregistry/dockerregistry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,15 @@ func ExecutePruner(configFile io.Reader, dryRun bool) {
log.Fatalf("error creating registry: %s", err)
}

stats, err := prune.Prune(ctx, storageDriver, registry, registryClient, dryRun)
var pruner prune.Pruner

if dryRun {
pruner = &prune.DryRunPruner{}
} else {
pruner = &prune.RegistryPruner{storageDriver}
}

stats, err := prune.Prune(ctx, registry, registryClient, pruner)
if err != nil {
log.Error(err)
}
Expand Down
182 changes: 142 additions & 40 deletions pkg/dockerregistry/server/prune/prune.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,133 @@ import (
imageapiv1 "github.com/openshift/origin/pkg/image/apis/image/v1"
)

// Pruner defines a common set of operations for pruning
type Pruner interface {
DeleteRepository(ctx context.Context, reponame string) error
DeleteManifestLink(ctx context.Context, svc distribution.ManifestService, reponame string, dgst digest.Digest) error
DeleteBlob(ctx context.Context, dgst digest.Digest) error
}

// DryRunPruner prints information about each object that going to remove.
type DryRunPruner struct{}

var _ Pruner = &DryRunPruner{}

func (p *DryRunPruner) DeleteRepository(ctx context.Context, reponame string) error {
logger := context.GetLogger(ctx)
logger.Printf("Would delete repository: %s", reponame)
return nil
}

func (p *DryRunPruner) DeleteManifestLink(ctx context.Context, svc distribution.ManifestService, reponame string, dgst digest.Digest) error {
logger := context.GetLogger(ctx)
logger.Printf("Would delete manifest link: %s@%s", reponame, dgst)
return nil
}

func (p *DryRunPruner) DeleteBlob(ctx context.Context, dgst digest.Digest) error {
logger := context.GetLogger(ctx)
logger.Printf("Would delete blob: %s", dgst)
return nil
}

// RegistryPruner deletes objects.
type RegistryPruner struct {
StorageDriver driver.StorageDriver
}

var _ Pruner = &RegistryPruner{}

// DeleteRepository removes a repository directory from the storage
func (p *RegistryPruner) DeleteRepository(ctx context.Context, reponame string) error {
logger := context.GetLogger(ctx)
vacuum := storage.NewVacuum(ctx, p.StorageDriver)

// Log message will be generated by RemoveRepository with loglevel=info.
if err := vacuum.RemoveRepository(reponame); err != nil {
return fmt.Errorf("unable to remove the repository %s: %v", reponame, err)
}

return nil
}

// DeleteManifestLink removes a manifest link from the storage
func (p *RegistryPruner) DeleteManifestLink(ctx context.Context, svc distribution.ManifestService, reponame string, dgst digest.Digest) error {
logger := context.GetLogger(ctx)

logger.Printf("Deleting manifest link: %s@%s", reponame, dgst)
if err := svc.Delete(ctx, dgst); err != nil {
return fmt.Errorf("failed to delete the manifest link %s@%s: %v", reponame, dgst, err)
}

return nil
}

// DeleteBlob removes a blob from the storage
func (p *RegistryPruner) DeleteBlob(ctx context.Context, dgst digest.Digest) error {
vacuum := storage.NewVacuum(ctx, p.StorageDriver)

// Log message will be generated by RemoveBlob with loglevel=info.
if err := vacuum.RemoveBlob(string(dgst)); err != nil {
return fmt.Errorf("failed to delete the blob %s: %v", dgst, err)
}

return nil
}

// garbageCollector holds objects for later deletion. If the object is replaced,
// then the previous one will be deleted.
type garbageCollector struct {
Pruner Pruner
Ctx context.Context

repoName string

manifestService distribution.ManifestService
manifestRepo string
manifestLink digest.Digest
}

func (gc *garbageCollector) AddRepository(repoName string) error {
// If the place is occupied, then it is necessary to clean it.
if err := gc.Collect(); err != nil {
return err
}

gc.repoName = repoName

return nil
}

func (gc *garbageCollector) AddManifestLink(svc distribution.ManifestService, repoName string, dgst digest.Digest) error {
// If the place is occupied, then it is necessary to clean it.
if err := gc.Collect(); err != nil {
return err
}

gc.manifestService = svc
gc.manifestRepo = repoName
gc.manifestLink = dgst

return nil
}

func (gc *garbageCollector) Collect() error {
if len(gc.manifestLink) > 0 {
if err := gc.Pruner.DeleteManifestLink(gc.Ctx, gc.manifestService, gc.manifestRepo, gc.manifestLink); err != nil {
return err
}
gc.manifestLink = ""
}
if len(gc.repoName) > 0 {
if err := gc.Pruner.DeleteRepository(gc.Ctx, gc.repoName); err != nil {
return err
}
gc.repoName = ""
}
return nil
}

func imageStreamHasManifestDigest(is *imageapiv1.ImageStream, dgst digest.Digest) bool {
for _, tagEventList := range is.Status.Tags {
for _, tagEvent := range tagEventList.Items {
Expand All @@ -42,7 +169,7 @@ type Summary struct {
//
// TODO(dmage): remove layer links to a blob if the blob is removed or it doesn't belong to the ImageStream.
// TODO(dmage): keep young blobs (docker/distribution#2297).
func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, registryClient client.RegistryClient, dryRun bool) (Summary, error) {
func Prune(ctx context.Context, registry distribution.Namespace, registryClient client.RegistryClient, pruner Pruner) (Summary, error) {
logger := context.GetLogger(ctx)

repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
Expand Down Expand Up @@ -84,7 +211,15 @@ func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry dis

var stats Summary

var reposToDelete []string
// The Enumerate calls a Stat() on each file or directory in the tree before call our handler.
// Therefore, we can not delete subdirectories from the handler. On some types of storage (S3),
// this can lead to an error in the Enumerate.
// We are waiting for the completion of our handler and perform deferred deletion of objects.
gc := &garbageCollector{
Ctx: ctx,
Pruner: pruner,
}

err = repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
logger.Debugln("Processing repository", repoName)

Expand All @@ -101,11 +236,7 @@ func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry dis
is, err := oc.ImageStreams(ref.Namespace).Get(ref.Name, metav1.GetOptions{})
if kerrors.IsNotFound(err) {
logger.Printf("The image stream %s/%s is not found, will remove the whole repository", ref.Namespace, ref.Name)

// We cannot delete the repository at this point, because it would break Enumerate.
reposToDelete = append(reposToDelete, repoName)

return nil
return gc.AddRepository(repoName)
} else if err != nil {
return fmt.Errorf("failed to get the image stream %s: %v", repoName, err)
}
Expand All @@ -131,17 +262,7 @@ func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry dis
return nil
}

if dryRun {
logger.Printf("Would delete manifest link: %s@%s", repoName, dgst)
return nil
}

logger.Printf("Deleting manifest link: %s@%s", repoName, dgst)
if err := manifestService.Delete(ctx, dgst); err != nil {
return fmt.Errorf("failed to delete the manifest link %s@%s: %v", repoName, dgst, err)
}

return nil
return gc.AddManifestLink(manifestService, repoName, dgst)
})
if e, ok := err.(driver.PathNotFoundError); ok {
logger.Printf("Skipped manifest link pruning for the repository %s: %v", repoName, e)
Expand All @@ -158,18 +279,8 @@ func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry dis
return stats, err
}

vacuum := storage.NewVacuum(ctx, storageDriver)

logger.Debugln("Removing repositories")
for _, repoName := range reposToDelete {
if dryRun {
logger.Printf("Would delete repository: %s", repoName)
continue
}

if err = vacuum.RemoveRepository(repoName); err != nil {
return stats, fmt.Errorf("unable to remove the repository %s: %v", repoName, err)
}
if err := gc.Collect(); err != nil {
return stats, err
}

logger.Debugln("Processing blobs")
Expand All @@ -188,16 +299,7 @@ func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry dis
stats.Blobs++
stats.DiskSpace += desc.Size

if dryRun {
logger.Printf("Would delete blob: %s", dgst)
return nil
}

if err := vacuum.RemoveBlob(string(dgst)); err != nil {
return fmt.Errorf("failed to delete the blob %s: %v", dgst, err)
}

return nil
return pruner.DeleteBlob(ctx, dgst)
})
return stats, err
}

0 comments on commit 79afcb3

Please sign in to comment.