From d1935e7e21994c650a344d8d5a5449762464a6fe Mon Sep 17 00:00:00 2001 From: Austin Vazquez Date: Sat, 17 Aug 2024 04:30:05 +0000 Subject: [PATCH] Add image prune --filter support Signed-off-by: Austin Vazquez --- cmd/nerdctl/image_prune.go | 10 ++++ cmd/nerdctl/image_prune_test.go | 50 +++++++++++++++++++ docs/command-reference.md | 5 +- pkg/api/types/image_types.go | 2 + pkg/cmd/image/prune.go | 50 ++++++++++--------- pkg/imgutil/filtering.go | 71 +++++++++++++++++++++++++++ pkg/imgutil/filtering_test.go | 85 +++++++++++++++++++++++++++++++++ pkg/imgutil/imgutil.go | 49 +++++++++++++++++++ 8 files changed, 294 insertions(+), 28 deletions(-) diff --git a/cmd/nerdctl/image_prune.go b/cmd/nerdctl/image_prune.go index 56dd8797f1a..4508efaaa10 100644 --- a/cmd/nerdctl/image_prune.go +++ b/cmd/nerdctl/image_prune.go @@ -38,6 +38,7 @@ func newImagePruneCommand() *cobra.Command { } imagePruneCommand.Flags().BoolP("all", "a", false, "Remove all unused images, not just dangling ones") + imagePruneCommand.Flags().StringSlice("filter", []string{}, "Filter output based on conditions provided") imagePruneCommand.Flags().BoolP("force", "f", false, "Do not prompt for confirmation") return imagePruneCommand } @@ -52,6 +53,14 @@ func processImagePruneOptions(cmd *cobra.Command) (types.ImagePruneOptions, erro return types.ImagePruneOptions{}, err } + var filters []string + if cmd.Flags().Changed("filter") { + filters, err = cmd.Flags().GetStringSlice("filter") + if err != nil { + return types.ImagePruneOptions{}, err + } + } + force, err := cmd.Flags().GetBool("force") if err != nil { return types.ImagePruneOptions{}, err @@ -61,6 +70,7 @@ func processImagePruneOptions(cmd *cobra.Command) (types.ImagePruneOptions, erro Stdout: cmd.OutOrStdout(), GOptions: globalOptions, All: all, + Filters: filters, Force: force, }, err } diff --git a/cmd/nerdctl/image_prune_test.go b/cmd/nerdctl/image_prune_test.go index e2babfbde24..e3025896ddd 100644 --- a/cmd/nerdctl/image_prune_test.go +++ b/cmd/nerdctl/image_prune_test.go @@ -71,3 +71,53 @@ func TestImagePruneAll(t *testing.T) { base.Cmd("image", "prune", "--force", "--all").AssertOutContains(imageName) base.Cmd("images").AssertOutNotContains(imageName) } + +func TestImagePruneFilterLabel(t *testing.T) { + testutil.RequiresBuild(t) + testutil.RegisterBuildCacheCleanup(t) + + base := testutil.NewBase(t) + imageName := testutil.Identifier(t) + t.Cleanup(func() { base.Cmd("rmi", "--force", imageName) }) + + dockerfile := fmt.Sprintf(`FROM %s +CMD ["echo", "nerdctl-test-image-prune-filter-label"] +LABEL foo=bar +LABEL version=0.1`, testutil.CommonImage) + + buildCtx := createBuildContext(t, dockerfile) + + base.Cmd("build", "-t", imageName, buildCtx).AssertOK() + base.Cmd("images", "--all").AssertOutContains(imageName) + + base.Cmd("image", "prune", "--force", "--all", "--filter", "label=foo=baz").AssertOK() + base.Cmd("images", "--all").AssertOutContains(imageName) + + base.Cmd("image", "prune", "--force", "--all", "--filter", "label=foo=bar").AssertOK() + base.Cmd("images", "--all").AssertOutNotContains(imageName) +} + +func TestImagePruneFilterUntil(t *testing.T) { + testutil.RequiresBuild(t) + testutil.RegisterBuildCacheCleanup(t) + + base := testutil.NewBase(t) + imageName := testutil.Identifier(t) + t.Cleanup(func() { base.Cmd("rmi", "--force", imageName) }) + + dockerfile := fmt.Sprintf(`FROM %s +CMD ["echo", "nerdctl-test-image-prune-filter-label"] +LABEL foo=bar +LABEL version=0.1`, testutil.CommonImage) + + buildCtx := createBuildContext(t, dockerfile) + + base.Cmd("build", "-t", imageName, buildCtx).AssertOK() + base.Cmd("images").AssertOutContains(imageName) + + base.Cmd("image", "prune", "--force", "--all", "--filter", "until=12h").AssertOK() + base.Cmd("images", "--all").AssertOutContains(imageName) + + base.Cmd("image", "prune", "--force", "--all", "--filter", "until=100ms").AssertOK() + base.Cmd("images", "--all").AssertOutNotContains(imageName) +} diff --git a/docs/command-reference.md b/docs/command-reference.md index 9106aa76397..4508b65f8c8 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -886,10 +886,11 @@ Usage: `nerdctl image prune [OPTIONS]` Flags: - :whale: `-a, --all`: Remove all unused images, not just dangling ones +- :whale: `-f, --filter`: Filter the images. + - :whale: `--filter=until=`: Images created before given date formatted timestamps or Go duration strings. Currently does not support Unix timestamps. + - :whale: `--filter=label=`: Matches images based on the presence of a label alone or a label and a value - :whale: `-f, --force`: Do not prompt for confirmation -Unimplemented `docker image prune` flags: `--filter` - ### :nerd_face: nerdctl image convert Convert an image format. diff --git a/pkg/api/types/image_types.go b/pkg/api/types/image_types.go index 40015e8a24c..30e0d65c31c 100644 --- a/pkg/api/types/image_types.go +++ b/pkg/api/types/image_types.go @@ -236,6 +236,8 @@ type ImagePruneOptions struct { GOptions GlobalCommandOptions // All Remove all unused images, not just dangling ones. All bool + // Filters output based on conditions provided for the --filter argument + Filters []string // Force will not prompt for confirmation. Force bool } diff --git a/pkg/cmd/image/prune.go b/pkg/cmd/image/prune.go index 159d2d6c8b1..da29fbdb486 100644 --- a/pkg/cmd/image/prune.go +++ b/pkg/cmd/image/prune.go @@ -34,45 +34,43 @@ import ( // Prune will remove all dangling images. If all is specified, will also remove all images not referenced by any container. func Prune(ctx context.Context, client *containerd.Client, options types.ImagePruneOptions) error { var ( - imageStore = client.ImageService() - contentStore = client.ContentStore() - containerStore = client.ContainerService() + imageStore = client.ImageService() + contentStore = client.ContentStore() ) - imageList, err := imageStore.List(ctx) - if err != nil { - return err - } - - var filteredImages []images.Image + var ( + imagesToBeRemoved []images.Image + err error + ) - if options.All { - containerList, err := containerStore.List(ctx) + filters := []imgutil.Filter{} + if len(options.Filters) > 0 { + parsedFilters, err := imgutil.ParseFilters(options.Filters) if err != nil { return err } - usedImages := make(map[string]struct{}) - for _, container := range containerList { - usedImages[container.Image] = struct{}{} + if len(parsedFilters.Labels) > 0 { + filters = append(filters, imgutil.FilterByLabel(ctx, client, parsedFilters.Labels)) } - - for _, image := range imageList { - if _, ok := usedImages[image.Name]; ok { - continue - } - - filteredImages = append(filteredImages, image) + if len(parsedFilters.Until) > 0 { + filters = append(filters, imgutil.FilterUntil(parsedFilters.Until)) } + } + + if options.All { + // Remove all unused images; not just dangling ones + imagesToBeRemoved, err = imgutil.GetUnusedImages(ctx, client, filters...) } else { - filteredImages, err = imgutil.FilterDanglingImages()(imageList) - if err != nil { - return err - } + // Remove dangling images only + imagesToBeRemoved, err = imgutil.GetDanglingImages(ctx, client, filters...) + } + if err != nil { + return err } delOpts := []images.DeleteOpt{images.SynchronousDelete()} removedImages := make(map[string][]digest.Digest) - for _, image := range filteredImages { + for _, image := range imagesToBeRemoved { digests, err := image.RootFS(ctx, contentStore, platforms.DefaultStrict()) if err != nil { log.G(ctx).WithError(err).Warnf("failed to enumerate rootfs") diff --git a/pkg/imgutil/filtering.go b/pkg/imgutil/filtering.go index fa35a909970..05d178a4fe5 100644 --- a/pkg/imgutil/filtering.go +++ b/pkg/imgutil/filtering.go @@ -18,6 +18,7 @@ package imgutil import ( "context" + "errors" "fmt" "regexp" "strings" @@ -35,15 +36,23 @@ import ( const ( FilterBeforeType = "before" FilterSinceType = "since" + FilterUntilType = "until" FilterLabelType = "label" FilterReferenceType = "reference" FilterDanglingType = "dangling" ) +var ( + errMultipleUntilFilters = errors.New("more than one until filter provided") + errNoUntilTimestamp = errors.New("no until timestamp provided") + errUnparsableUntilTimestamp = errors.New("unable to parse until timestamp") +) + // Filters contains all types of filters to filter images. type Filters struct { Before []string Since []string + Until string Labels map[string]string Reference []string Dangling *bool @@ -85,6 +94,13 @@ func ParseFilters(filters []string) (*Filters, error) { } f.Since = append(f.Since, fmt.Sprintf("name==%s", canonicalRef.String())) f.Since = append(f.Since, fmt.Sprintf("name==%s", tempFilterToken[1])) + } else if tempFilterToken[0] == FilterUntilType { + if len(tempFilterToken[0]) == 0 { + return nil, errNoUntilTimestamp + } else if len(f.Until) > 0 { + return nil, errMultipleUntilFilters + } + f.Until = tempFilterToken[1] } else if tempFilterToken[0] == FilterLabelType { // To support filtering labels by keys. f.Labels[tempFilterToken[1]] = "" @@ -161,6 +177,57 @@ func FilterByCreatedAt(ctx context.Context, client *containerd.Client, before [] } } +// FilterUntil filters images created before the provided timestamp. +func FilterUntil(until string) Filter { + return func(imageList []images.Image) ([]images.Image, error) { + if len(until) == 0 { + return []images.Image{}, errNoUntilTimestamp + } + + var ( + parsedTime time.Time + err error + ) + + type parseUntilFunc func(string) (time.Time, error) + parsingFuncs := []parseUntilFunc{ + func(until string) (time.Time, error) { + return time.Parse(time.RFC3339, until) + }, + func(until string) (time.Time, error) { + return time.Parse(time.RFC3339Nano, until) + }, + func(until string) (time.Time, error) { + return time.Parse(time.DateOnly, until) + }, + func(until string) (time.Time, error) { + // Go duration strings + d, err := time.ParseDuration(until) + if err != nil { + return time.Time{}, err + } + return time.Now().Add(-d), nil + }, + } + + for _, parse := range parsingFuncs { + parsedTime, err = parse(until) + if err != nil { + continue + } + break + } + + if err != nil { + return []images.Image{}, errUnparsableUntilTimestamp + } + + return filter(imageList, func(i images.Image) (bool, error) { + return imageCreatedBefore(i, parsedTime), nil + }) + } +} + // FilterByLabel filters an image list based on labels applied to the image's config specification for the platform. // Any matching label will include the image in the list. func FilterByLabel(ctx context.Context, client *containerd.Client, labels map[string]string) Filter { @@ -221,6 +288,10 @@ func imageCreatedBetween(image images.Image, min time.Time, max time.Time) bool return image.CreatedAt.After(min) && image.CreatedAt.Before(max) } +func imageCreatedBefore(image images.Image, max time.Time) bool { + return image.CreatedAt.Before(max) +} + func matchesAllLabels(imageCfgLabels map[string]string, filterLabels map[string]string) bool { var matches int for lk, lv := range filterLabels { diff --git a/pkg/imgutil/filtering_test.go b/pkg/imgutil/filtering_test.go index 9ef4746ed41..f2645f87f7a 100644 --- a/pkg/imgutil/filtering_test.go +++ b/pkg/imgutil/filtering_test.go @@ -108,6 +108,23 @@ func TestApplyFilters(t *testing.T) { }, }, }, + { + name: "ReturnErrorAndEmptyListOnFilterError", + images: []images.Image{ + { + Name: ":", + }, + { + Name: "docker.io/library/hello-world:latest", + }, + }, + filters: []Filter{ + FilterDanglingImages(), + FilterUntil(""), + }, + expectedImages: []images.Image{}, + expectedErr: errNoUntilTimestamp, + }, } for _, test := range tests { @@ -124,6 +141,74 @@ func TestApplyFilters(t *testing.T) { } } +func TestFilterUntil(t *testing.T) { + now := time.Now().UTC() + + tests := []struct { + name string + until string + images []images.Image + expectedImages []images.Image + expectedErr error + }{ + { + name: "EmptyTimestampReturnsError", + until: "", + images: []images.Image{}, + expectedImages: []images.Image{}, + expectedErr: errNoUntilTimestamp, + }, + { + name: "UnparseableTimestampReturnsError", + until: "-2006-01-02T15:04:05Z07:00", + images: []images.Image{}, + expectedImages: []images.Image{}, + expectedErr: errUnparsableUntilTimestamp, + }, + { + name: "ImagesOlderThan3Hours(Go duration)", + until: "3h", + images: []images.Image{ + { + Name: "image:yesterday", + CreatedAt: now.Add(-24 * time.Hour), + }, + { + Name: "image:today", + CreatedAt: now.Add(-12 * time.Hour), + }, + { + Name: "image:latest", + CreatedAt: now, + }, + }, + expectedImages: []images.Image{ + { + Name: "image:yesterday", + CreatedAt: now.Add(-24 * time.Hour), + }, + { + Name: "image:today", + CreatedAt: now.Add(-12 * time.Hour), + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actualImages, err := FilterUntil(test.until)(test.images) + if test.expectedErr == nil { + assert.NilError(t, err) + } else { + assert.ErrorIs(t, err, test.expectedErr) + } + assert.Equal(t, len(actualImages), len(test.expectedImages)) + assert.DeepEqual(t, actualImages, test.expectedImages) + }) + } +} + func TestFilterByReference(t *testing.T) { tests := []struct { name string diff --git a/pkg/imgutil/imgutil.go b/pkg/imgutil/imgutil.go index 5406e83858d..3b042c9fb63 100644 --- a/pkg/imgutil/imgutil.go +++ b/pkg/imgutil/imgutil.go @@ -423,3 +423,52 @@ func UnpackedImageSize(ctx context.Context, s snapshots.Snapshotter, img contain return total.Size, err } + +// GetUnusedImages returns the list of all images which are not referenced by a container. +func GetUnusedImages(ctx context.Context, client *containerd.Client, filters ...Filter) ([]images.Image, error) { + var ( + imageStore = client.ImageService() + containerStore = client.ContainerService() + ) + + containers, err := containerStore.List(ctx) + if err != nil { + return []images.Image{}, err + } + + usedImages := make(map[string]struct{}) + for _, container := range containers { + usedImages[container.Image] = struct{}{} + } + + allImages, err := imageStore.List(ctx) + if err != nil { + return []images.Image{}, err + } + + unusedImages := make([]images.Image, 0, len(allImages)) + for _, image := range allImages { + if _, ok := usedImages[image.Name]; ok { + continue + } + unusedImages = append(unusedImages, image) + } + + return ApplyFilters(unusedImages, filters...) +} + +// GetDanglingImages returns the list of all images which are not tagged. +func GetDanglingImages(ctx context.Context, client *containerd.Client, filters ...Filter) ([]images.Image, error) { + var ( + imageStore = client.ImageService() + ) + + allImages, err := imageStore.List(ctx) + if err != nil { + return []images.Image{}, err + } + + filters = append([]Filter{FilterDanglingImages()}, filters...) + + return ApplyFilters(allImages, filters...) +}