-
Notifications
You must be signed in to change notification settings - Fork 527
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat: implement prune flag * address changes * revert * boilerplate * rename * boilerplate * Update .gitattributes --------- Co-authored-by: Jason Hall <jason@chainguard.dev>
- Loading branch information
Showing
16 changed files
with
376 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
// Copyright 2018 Google LLC All Rights Reserved. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package cmd | ||
|
||
import ( | ||
"fmt" | ||
"os" | ||
|
||
"github.com/google/go-containerregistry/pkg/v1/layout" | ||
"github.com/spf13/cobra" | ||
) | ||
|
||
func NewCmdLayout() *cobra.Command { | ||
cmd := &cobra.Command{ | ||
Use: "layout", | ||
} | ||
cmd.AddCommand(newCmdGc()) | ||
return cmd | ||
} | ||
|
||
// NewCmdGc creates a new cobra.Command for the pull subcommand. | ||
func newCmdGc() *cobra.Command { | ||
cmd := &cobra.Command{ | ||
Use: "gc OCI-LAYOUT", | ||
Short: "Garbage collect unreferenced blobs in a local oci-layout", | ||
Args: cobra.ExactArgs(1), | ||
Hidden: true, // TODO: promote to public once theres some milage | ||
RunE: func(_ *cobra.Command, args []string) error { | ||
path := args[0] | ||
|
||
p, err := layout.FromPath(path) | ||
|
||
if err != nil { | ||
return err | ||
} | ||
|
||
blobs, err := p.GarbageCollect() | ||
if err != nil { | ||
return err | ||
} | ||
|
||
for _, blob := range blobs { | ||
if err := p.RemoveBlob(blob); err != nil { | ||
return err | ||
} | ||
fmt.Fprintf(os.Stderr, "garbage collecting: %s\n", blob.String()) | ||
} | ||
|
||
return nil | ||
}, | ||
} | ||
|
||
return cmd | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,137 @@ | ||
// Copyright 2018 Google LLC All Rights Reserved. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
// This is an EXPERIMENTAL package, and may change in arbitrary ways without notice. | ||
package layout | ||
|
||
import ( | ||
"fmt" | ||
"io/fs" | ||
"path/filepath" | ||
"strings" | ||
|
||
v1 "github.com/google/go-containerregistry/pkg/v1" | ||
) | ||
|
||
// GarbageCollect removes unreferenced blobs from the oci-layout | ||
// | ||
// This is an experimental api, and not subject to any stability guarantees | ||
// We may abandon it at any time, without prior notice. | ||
// Deprecated: Use it at your own risk! | ||
func (l Path) GarbageCollect() ([]v1.Hash, error) { | ||
idx, err := l.ImageIndex() | ||
if err != nil { | ||
return nil, err | ||
} | ||
blobsToKeep := map[string]bool{} | ||
if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil { | ||
return nil, err | ||
} | ||
blobsDir := l.path("blobs") | ||
removedBlobs := []v1.Hash{} | ||
|
||
err = filepath.WalkDir(blobsDir, func(path string, d fs.DirEntry, err error) error { | ||
if err != nil { | ||
return err | ||
} | ||
|
||
if d.IsDir() { | ||
return nil | ||
} | ||
|
||
rel, err := filepath.Rel(blobsDir, path) | ||
if err != nil { | ||
return err | ||
} | ||
hashString := strings.Replace(rel, "/", ":", 1) | ||
if present := blobsToKeep[hashString]; !present { | ||
h, err := v1.NewHash(hashString) | ||
if err != nil { | ||
return err | ||
} | ||
removedBlobs = append(removedBlobs, h) | ||
} | ||
return nil | ||
}) | ||
|
||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
return removedBlobs, nil | ||
} | ||
|
||
func (l Path) garbageCollectImageIndex(index v1.ImageIndex, blobsToKeep map[string]bool) error { | ||
idxm, err := index.IndexManifest() | ||
if err != nil { | ||
return err | ||
} | ||
|
||
h, err := index.Digest() | ||
if err != nil { | ||
return err | ||
} | ||
|
||
blobsToKeep[h.String()] = true | ||
|
||
for _, descriptor := range idxm.Manifests { | ||
if descriptor.MediaType.IsImage() { | ||
img, err := index.Image(descriptor.Digest) | ||
if err != nil { | ||
return err | ||
} | ||
if err := l.garbageCollectImage(img, blobsToKeep); err != nil { | ||
return err | ||
} | ||
} else if descriptor.MediaType.IsIndex() { | ||
idx, err := index.ImageIndex(descriptor.Digest) | ||
if err != nil { | ||
return err | ||
} | ||
if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil { | ||
return err | ||
} | ||
} else { | ||
return fmt.Errorf("gc: unknown media type: %s", descriptor.MediaType) | ||
} | ||
} | ||
return nil | ||
} | ||
|
||
func (l Path) garbageCollectImage(image v1.Image, blobsToKeep map[string]bool) error { | ||
h, err := image.Digest() | ||
if err != nil { | ||
return err | ||
} | ||
blobsToKeep[h.String()] = true | ||
|
||
h, err = image.ConfigName() | ||
if err != nil { | ||
return err | ||
} | ||
blobsToKeep[h.String()] = true | ||
|
||
ls, err := image.Layers() | ||
if err != nil { | ||
return err | ||
} | ||
for _, l := range ls { | ||
h, err := l.Digest() | ||
if err != nil { | ||
return err | ||
} | ||
blobsToKeep[h.String()] = true | ||
} | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,96 @@ | ||
// Copyright 2018 Google LLC All Rights Reserved. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package layout | ||
|
||
import ( | ||
"path/filepath" | ||
"testing" | ||
) | ||
|
||
var ( | ||
gcIndexPath = filepath.Join("testdata", "test_gc_index") | ||
gcIndexBlobHash = "sha256:492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0" | ||
gcUnknownMediaTypePath = filepath.Join("testdata", "test_gc_image_unknown_mediatype") | ||
gcUnknownMediaTypeErr = "gc: unknown media type: application/vnd.oci.descriptor.v1+json" | ||
gcTestOneImagePath = filepath.Join("testdata", "test_index_one_image") | ||
gcTestIndexMediaTypePath = filepath.Join("testdata", "test_index_media_type") | ||
) | ||
|
||
func TestGcIndex(t *testing.T) { | ||
lp, err := FromPath(gcIndexPath) | ||
if err != nil { | ||
t.Fatalf("FromPath() = %v", err) | ||
} | ||
|
||
removed, err := lp.GarbageCollect() | ||
if err != nil { | ||
t.Fatalf("GarbageCollect() = %v", err) | ||
} | ||
|
||
if len(removed) != 1 { | ||
t.Fatalf("expected to have only one gc-able blob") | ||
} | ||
if removed[0].String() != gcIndexBlobHash { | ||
t.Fatalf("wrong blob is gc-ed: expected '%s', got '%s'", gcIndexBlobHash, removed[0].String()) | ||
} | ||
} | ||
|
||
func TestGcOneImage(t *testing.T) { | ||
lp, err := FromPath(gcTestOneImagePath) | ||
if err != nil { | ||
t.Fatalf("FromPath() = %v", err) | ||
} | ||
|
||
removed, err := lp.GarbageCollect() | ||
if err != nil { | ||
t.Fatalf("GarbageCollect() = %v", err) | ||
} | ||
|
||
if len(removed) != 0 { | ||
t.Fatalf("expected to have to gc-able blobs") | ||
} | ||
} | ||
|
||
func TestGcIndexMediaType(t *testing.T) { | ||
lp, err := FromPath(gcTestIndexMediaTypePath) | ||
if err != nil { | ||
t.Fatalf("FromPath() = %v", err) | ||
} | ||
|
||
removed, err := lp.GarbageCollect() | ||
if err != nil { | ||
t.Fatalf("GarbageCollect() = %v", err) | ||
} | ||
|
||
if len(removed) != 0 { | ||
t.Fatalf("expected to have to gc-able blobs") | ||
} | ||
} | ||
|
||
func TestGcUnknownMediaType(t *testing.T) { | ||
lp, err := FromPath(gcUnknownMediaTypePath) | ||
if err != nil { | ||
t.Fatalf("FromPath() = %v", err) | ||
} | ||
|
||
_, err = lp.GarbageCollect() | ||
if err == nil { | ||
t.Fatalf("expected GarbageCollect to return err but did not") | ||
} | ||
|
||
if err.Error() != gcUnknownMediaTypeErr { | ||
t.Fatalf("expected error '%s', got '%s'", gcUnknownMediaTypeErr, err.Error()) | ||
} | ||
} |
10 changes: 10 additions & 0 deletions
10
pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/index.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
{ | ||
"schemaVersion": 2, | ||
"manifests": [ | ||
{ | ||
"mediaType": "application/vnd.oci.descriptor.v1+json", | ||
"size": 423, | ||
"digest": "sha256:32589985702551b6c56033bb3334432a0a513bf9d6aceda0f67c42b003850720" | ||
} | ||
] | ||
} |
3 changes: 3 additions & 0 deletions
3
pkg/v1/layout/testdata/test_gc_image_unknown_mediatype/oci-layout
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{ | ||
"imageLayoutVersion": "1.0.0" | ||
} |
13 changes: 13 additions & 0 deletions
13
...st_gc_index/blobs/sha256/05f95b26ed10668b7183c1e2da98610e91372fa9f510046d4ce5812addad86b5
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
{ | ||
"schemaVersion": 2, | ||
"manifests": [ | ||
{ | ||
"mediaType": "application/vnd.oci.image.manifest.v1+json", | ||
"size": 423, | ||
"digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", | ||
"annotations": { | ||
"org.opencontainers.image.ref.name": "1" | ||
} | ||
} | ||
] | ||
} |
13 changes: 13 additions & 0 deletions
13
...st_gc_index/blobs/sha256/2b29a2b8dea3af91ea7d0154be1da0c92d55ddd098540930fc8d3db7de377fdb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
{ | ||
"schemaVersion": 2, | ||
"manifests": [ | ||
{ | ||
"mediaType": "application/vnd.oci.image.manifest.v1+json", | ||
"size": 423, | ||
"digest": "sha256:eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650", | ||
"annotations": { | ||
"org.opencontainers.image.ref.name": "4" | ||
} | ||
} | ||
] | ||
} |
Binary file added
BIN
+114 Bytes
...st_gc_index/blobs/sha256/492b89b9dd3cda4596f94916d17f6901455fb8bd7f4c5a2a90df8d39c90f48a0
Binary file not shown.
1 change: 1 addition & 0 deletions
1
...st_gc_index/blobs/sha256/6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
{"architecture": "amd64", "author": "Bazel", "config": {}, "created": "1970-01-01T00:00:00Z", "history": [{"author": "Bazel", "created": "1970-01-01T00:00:00Z", "created_by": "bazel build ..."}], "os": "linux", "rootfs": {"diff_ids": ["sha256:8897395fd26dc44ad0e2a834335b33198cb41ac4d98dfddf58eced3853fa7b17"], "type": "layers"}} |
Binary file added
BIN
+167 Bytes
...st_gc_index/blobs/sha256/dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b
Binary file not shown.
1 change: 1 addition & 0 deletions
1
...st_gc_index/blobs/sha256/eebff607b1628d67459b0596643fc07de70d702eccf030f0bc7bb6fc2b278650
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":330,"digest":"sha256:6e0b05049ed9c17d02e1a55e80d6599dbfcce7f4f4b022e3c673e685789c470e"},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","size":167,"digest":"sha256:dc52c6e48a1d51a96047b059f16889bc889c4b4c28f3b36b3f93187f62fc0b2b"}]} |
Oops, something went wrong.