From c25cad4f5664f53aa4c12ea33462ec553b79181e Mon Sep 17 00:00:00 2001 From: Will Lamm Date: Thu, 21 Dec 2023 11:30:51 -0600 Subject: [PATCH 1/4] pull from ECR and Grafana --- Gopkg.lock | 68 ------------- Gopkg.toml | 34 ------- go.mod | 36 +++++++ go.sum | 72 ++++++++++++++ main.go | 286 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 392 insertions(+), 104 deletions(-) delete mode 100644 Gopkg.lock delete mode 100644 Gopkg.toml create mode 100644 go.mod create mode 100644 go.sum diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index cfb10c6..0000000 --- a/Gopkg.lock +++ /dev/null @@ -1,68 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - name = "github.com/aws/aws-sdk-go" - packages = [ - "aws", - "aws/awserr", - "aws/awsutil", - "aws/client", - "aws/client/metadata", - "aws/corehandlers", - "aws/credentials", - "aws/credentials/ec2rolecreds", - "aws/credentials/endpointcreds", - "aws/credentials/stscreds", - "aws/defaults", - "aws/ec2metadata", - "aws/endpoints", - "aws/request", - "aws/session", - "aws/signer/v4", - "internal/shareddefaults", - "private/protocol", - "private/protocol/json/jsonutil", - "private/protocol/jsonrpc", - "private/protocol/query", - "private/protocol/query/queryutil", - "private/protocol/rest", - "private/protocol/xml/xmlutil", - "service/ecr", - "service/sts" - ] - revision = "960dd6034cbdac0e9716bfa66d91de88b286889f" - version = "v1.10.38" - -[[projects]] - name = "github.com/go-ini/ini" - packages = ["."] - revision = "c787282c39ac1fc618827141a1f762240def08a3" - -[[projects]] - name = "github.com/inconshreveable/mousetrap" - packages = ["."] - revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" - version = "v1.0" - -[[projects]] - name = "github.com/jmespath/go-jmespath" - packages = ["."] - revision = "bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d" - -[[projects]] - name = "github.com/spf13/cobra" - packages = ["."] - revision = "b78744579491c1ceeaaa3b40205e56b0591b93a3" - -[[projects]] - name = "github.com/spf13/pflag" - packages = ["."] - revision = "7aff26db30c1be810f9de5038ec5ef96ac41fd7c" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "62fd9b202c5d75597521500603c504eb2716d5f458cb587bb155c9d1f5fd516f" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index e25f421..0000000 --- a/Gopkg.toml +++ /dev/null @@ -1,34 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" -# -# [prune] -# non-go = false -# go-tests = true -# unused-packages = true - - -[[constraint]] - name = "github.com/aws/aws-sdk-go" - version = "1.10.38" - -[prune] - go-tests = true - unused-packages = true diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5cc3273 --- /dev/null +++ b/go.mod @@ -0,0 +1,36 @@ +module github.com/ltvco/ecr-cleaner + +go 1.21.4 + +require ( + github.com/aws/aws-sdk-go v1.48.7 + github.com/aws/aws-sdk-go-v2 v1.23.5 + github.com/aws/aws-sdk-go-v2/config v1.25.12 + github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3 + github.com/rs/zerolog v1.31.0 + github.com/sethpollack/ecr-cleaner v0.0.0-20180422155703-773c673bc438 + github.com/spf13/cobra v1.8.0 +) + +require ( + github.com/aws/aws-sdk-go-v2/credentials v1.16.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.3 // indirect + github.com/aws/smithy-go v1.18.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/sys v0.12.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) + +require github.com/jmespath/go-jmespath v0.4.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7503731 --- /dev/null +++ b/go.sum @@ -0,0 +1,72 @@ +github.com/aws/aws-sdk-go v1.48.7 h1:gDcOhmkohlNk20j0uWpko5cLBbwSkB+xpkshQO45F7Y= +github.com/aws/aws-sdk-go v1.48.7/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go-v2 v1.23.5 h1:xK6C4udTyDMd82RFvNkDQxtAd00xlzFUtX4fF2nMZyg= +github.com/aws/aws-sdk-go-v2 v1.23.5/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds= +github.com/aws/aws-sdk-go-v2/config v1.25.12 h1:mF4cMuNh/2G+d19nWnm1vJ/ak0qK6SbqF0KtSX9pxu0= +github.com/aws/aws-sdk-go-v2/config v1.25.12/go.mod h1:lOvvqtZP9p29GIjOTuA/76HiVk0c/s8qRcFRq2+E2uc= +github.com/aws/aws-sdk-go-v2/credentials v1.16.10 h1:VmRkuoKaGl2ZDNGkkRQgw80Hxj1Bb9a+bsT5shqlCwo= +github.com/aws/aws-sdk-go-v2/credentials v1.16.10/go.mod h1:WEn22lpd50buTs/TDqywytW5xQ2zPOMbYipIlqI6xXg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 h1:FZVFahMyZle6WcogZCOxo6D/lkDA2lqKIn4/ueUmVXw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9/go.mod h1:kjq7REMIkxdtcEC9/4BVXjOsNY5isz6jQbEgk6osRTU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 h1:8GVZIR0y6JRIUNSYI1xAMF4HDfV8H/bOsZ/8AD/uY5Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8/go.mod h1:rwBfu0SoUkBUZndVgPZKAD9Y2JigaZtRP68unRiYToQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 h1:ZE2ds/qeBkhk3yqYvS3CDCFNvd9ir5hMjlVStLZWrvM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8/go.mod h1:/lAPPymDYL023+TS6DJmjuL42nxix2AvEvfjqOBRODk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1 h1:uR9lXYjdPX0xY+NhvaJ4dD8rpSRz5VY81ccIIoNG+lw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.1/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3 h1:+sbyLjtAq0Xg9ZOQ2mBibklsGUyX6I2OfRTDsha9uU4= +github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3/go.mod h1:/m9MiYl5Ds0cZqy/bbeSUWxKLwTarGugjXxSgiXNQFc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3 h1:e3PCNeEaev/ZF01cQyNZgmYE9oYYePIMJs2mWSKG514= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.3/go.mod h1:gIeeNyaL8tIEqZrzAnTeyhHcE0yysCtcaP+N9kxLZ+E= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 h1:EamsKe+ZjkOQjDdHd86/JCEucjFKQ9T0atWKO4s2Lgs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8/go.mod h1:Q0vV3/csTpbkfKLI5Sb56cJQTCTtJ0ixdb7P+Wedqiw= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.3 h1:wKspi1zc2ZVcgZEu3k2Mt4zGKQSoZTftsoUTLsYPcVo= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.3/go.mod h1:zxk6y1X2KXThESWMS5CrKRvISD8mbIMab6nZrCGxDG0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3 h1:CxAHBS0BWSUqI7qzXHc2ZpTeHaM9JNnWJ9BN6Kmo2CY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3/go.mod h1:7Lt5mjQ8x5rVdKqg+sKKDeuwoszDJIIPmkd8BVsEdS0= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.3 h1:KfREzajmHCSYjCaMRtdLr9boUMA7KPpoPApitPlbNeo= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.3/go.mod h1:7Ld9eTqocTvJqqJ5K/orbSDwmGcpRdlDiLjz2DO+SL8= +github.com/aws/smithy-go v1.18.1 h1:pOdBTUfXNazOlxLrgeYalVnuTpKreACHtc62xLwIB3c= +github.com/aws/smithy-go v1.18.1/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sethpollack/ecr-cleaner v0.0.0-20180422155703-773c673bc438 h1:EIeAyisomPrBcw56k/u+KtQxH+LQR0jGhtMfoxXURMo= +github.com/sethpollack/ecr-cleaner v0.0.0-20180422155703-773c673bc438/go.mod h1:O84OMiYeGGrIBRWRQLHionF3n8iNQHhVTFlesPKUQH8= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 5c931e7..9fcee92 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,289 @@ package main -import "github.com/sethpollack/ecr-cleaner/cmd" +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ecr" + + // agent "github.com/ltvco/ltv-apm-modules-go/agent" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +type ImageInfo struct { + LastPushed time.Time + LastSeen time.Time + Digest string + RegistryUrl string + RepositoryName string + DeployedTag string + Tags []string + Cluster string + FullImagePath string +} + +type PostBody struct { + From string `json:"from"` + Queries []Queries `json:"queries"` + To string `json:"to"` +} +type Datasource struct { + UID string `json:"uid"` +} +type Queries struct { + Datasource Datasource `json:"datasource"` + Format string `json:"format"` + IntervalMs int `json:"intervalMs"` + MaxDataPoints int `json:"maxDataPoints"` + Expr string `json:"Expr"` + RefID string `json:"refId"` +} + +type Response struct { + Results struct { + A struct { + Status int `json:"status"` + Frames []struct { + Schema struct { + RefID string `json:"refId"` + Meta struct { + Type string `json:"type"` + TypeVersion []int `json:"typeVersion"` + Custom struct { + ResultType string `json:"resultType"` + } `json:"custom"` + ExecutedQueryString string `json:"executedQueryString"` + } `json:"meta"` + Fields []struct { + Name string `json:"name"` + Type string `json:"type"` + TypeInfo struct { + Frame string `json:"frame"` + } `json:"typeInfo"` + Config struct { + Interval int `json:"interval"` + } `json:"config,omitempty"` + Labels struct { + Name string `json:"__name__"` + Cluster string `json:"cluster"` + Container string `json:"container"` + ContainerID string `json:"container_id"` + Endpoint string `json:"endpoint"` + Image string `json:"image"` + ImageID string `json:"image_id"` + ImageSpec string `json:"image_spec"` + Instance string `json:"instance"` + Job string `json:"job"` + Namespace string `json:"namespace"` + Pod string `json:"pod"` + Service string `json:"service"` + UID string `json:"uid"` + } `json:"labels,omitempty"` + } `json:"fields"` + } `json:"schema"` + Data struct { + Values [][]int64 `json:"values"` + } `json:"data"` + } `json:"frames"` + } `json:"A"` + } `json:"results"` +} func main() { - cmd.Execute() + zerolog.SetGlobalLevel(zerolog.InfoLevel) + + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-east-1"), config.WithSharedConfigProfile("production")) + if err != nil { + log.Fatal().Err(err).Msg("unable to load SDK config") + } + + allImages := GetPrometheusImagesFromProfile() + + allEcrImages, err := GetECRImages(cfg) + if err != nil { + log.Fatal().Err(err).Msg("failed to get ECR images") + return + } + fmt.Printf("len(allImages): %v\n", len(allImages)) + for i, output := range allEcrImages { + untagged := 0 + for _, deets := range output.ImageDetails { + if deets.ImageTags == nil { + untagged++ + continue + } + // fmt.Printf("Index: %v imageName and tags: %v:%v\n", i, *deets.RepositoryName, deets.ImageTags) + // for _, image := range allImages { + + // if *deets.RepositoryName == image.RepositoryName { + // s, _ := json.MarshalIndent(deets, "", "\t") + // fmt.Printf("image.RepositoryName: %v\n", image.RepositoryName) + // fmt.Printf("s: %v\n", string(s)) + + // image.Tags = deets.ImageTags + // image.LastPushed = *deets.ImagePushedAt + // image.Digest = *deets.ImageDigest + + // } + // } + } + fmt.Printf("untagged in %v: %v\n", i, untagged) + } + fmt.Printf("before unique allImages: %v\n", len(allImages)) + allImages = GetUnique(allImages) + fmt.Printf("after unique allImages: %v\n", len(allImages)) + + for _, image := range allImages { + _, err := json.MarshalIndent(image, "", "\t") + if err != nil { + log.Error().Err(err).Msg("failed to marshalIndent json") + } + // fmt.Printf("s: %v\n", string(s)) + + // if image.LastPushed.Before(time.Now().AddDate(0, 0, -(daysOld))) { + // fmt.Printf("%v is older than %v days old\n", image.FullImagePath, daysOld) + // } + for _, ecr := range allEcrImages { + for _, deets := range ecr.ImageDetails { + // fmt.Printf("deets.ImageDigest: %v\n", *deets.ImageDigest) + deetsDigest := strings.Split(*deets.ImageDigest, ":")[1] + // fmt.Printf("deetsDigest: %v\n", deetsDigest) + // fmt.Printf("image.Digest: %v\n", image.Digest) + if deetsDigest == image.Digest { + + fmt.Printf("running image %v:%v pushed at %v\n", *deets.RepositoryName, deets.ImageTags, deets.ImagePushedAt) + continue + } + } + } + } + +} + +func GetPrometheusImagesFromProfile() []*ImageInfo { + allImages := make([]*ImageInfo, 0) + datasource := Datasource{ + UID: "W7Xb02Snk", + } + query := Queries{ + RefID: "A", + Expr: "kube_pod_container_info{cluster=\"promoted-walleye\"}", + + Datasource: datasource, + } + postBody := PostBody{ + From: "now-1h", + Queries: []Queries{query}, + To: "now", + } + buf, err := json.Marshal(postBody) + if err != nil { + log.Error().Err(err).Msg("failed to marshall json") + } + grafanaApiKey := os.Getenv("GRAFANA_API_KEY") + req, _ := http.NewRequest("POST", "https://grafana.ltvops.com/api/ds/query", bytes.NewBuffer(buf)) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", grafanaApiKey)) + req.Header.Add("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Error().Err(err).Msg("failed to call grafana") + } + defer resp.Body.Close() + + body := &Response{} + json.NewDecoder(resp.Body).Decode(body) + for _, item := range body.Results.A.Frames { + + for _, field := range item.Schema.Fields { + if field.Labels.Name == "kube_pod_container_info" { + + // var lastSeen time.Time + // if item.Data.Values[1][0] != 0 { + // lastSeen = time.Unix(item.Data.Values[1][0], 0) + // } + + registryUrl := strings.Split(strings.Split(field.Labels.Image, "@")[0], ":")[0] + if field.Labels.ImageID != "" { + // fmt.Printf("field.Labels.ImageID: %v\n", field.Labels.ImageID) + digest := strings.Split(strings.Split(field.Labels.ImageID, "@")[1], ":")[1] + newImage := ImageInfo{ + FullImagePath: field.Labels.Image, + RegistryUrl: registryUrl, + RepositoryName: strings.Split(registryUrl, "/")[len(strings.Split(registryUrl, "/"))-1], + DeployedTag: strings.Split(field.Labels.Image, ":")[1], + Digest: digest, + Cluster: field.Labels.Cluster, + } + allImages = append(allImages, &newImage) + } + + } + } + } + return allImages +} + +func GetECRImages(cfg aws.Config) ([]*ecr.DescribeImagesOutput, error) { + var allECRImages []*ecr.DescribeImagesOutput + client := ecr.NewFromConfig(cfg) + registry, _ := client.DescribeRegistry(context.Background(), &ecr.DescribeRegistryInput{}) + fmt.Printf("repos.RegistryId: %v\n", *registry.RegistryId) + repos, _ := client.DescribeRepositories(context.Background(), &ecr.DescribeRepositoriesInput{}) + for _, repo := range repos.Repositories { + if repo.RepositoryName != nil { + + input := &ecr.ListImagesInput{ + RepositoryName: repo.RepositoryName, + } + + images, err := client.ListImages(context.Background(), input) + if err != nil { + log.Error().Err(err).Msg("failed to list images") + return nil, err + } + if len(images.ImageIds) > 0 { + + describe, err := client.DescribeImages(context.Background(), &ecr.DescribeImagesInput{ + ImageIds: images.ImageIds, + RepositoryName: repo.RepositoryName, + }) + if err != nil { + log.Error().Err(err).Msg("failed to describe images") + return nil, err + } + allECRImages = append(allECRImages, describe) + + } + + } + } + return allECRImages, nil +} + +func GetUnique(all []*ImageInfo) []*ImageInfo { + var unique []*ImageInfo + for _, v := range all { + skip := false + for _, u := range unique { + if v.FullImagePath == u.FullImagePath { + skip = true + break + } + } + if !skip { + unique = append(unique, v) + } + } + return unique } From ce7ed13a6c62dc4273c53d86dcdfd852136b07c6 Mon Sep 17 00:00:00 2001 From: Will Lamm Date: Wed, 10 Jan 2024 13:30:03 -0600 Subject: [PATCH 2/4] feat: working prototype --- clean/clean.go | 331 ++++++++++++++++++++++++++++++++++++++++++++++++ clean/models.go | 82 ++++++++++++ cmd/root.go | 50 ++++---- ecr/ecr.go | 163 ------------------------ go.mod | 8 +- go.sum | 4 - main.go | 323 +++++++--------------------------------------- 7 files changed, 488 insertions(+), 473 deletions(-) create mode 100644 clean/clean.go create mode 100644 clean/models.go delete mode 100644 ecr/ecr.go diff --git a/clean/clean.go b/clean/clean.go new file mode 100644 index 0000000..1ae272f --- /dev/null +++ b/clean/clean.go @@ -0,0 +1,331 @@ +package clean + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "sort" + "strings" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ecr" + "github.com/aws/aws-sdk-go-v2/service/ecr/types" + "github.com/rs/zerolog/log" +) + +func CheckImageNotInUse(all []*ImageInfo, detail types.ImageDetail) bool { + for _, image := range all { + // _, err := json.MarshalIndent(image, "", "\t") + // if err != nil { + // log.Error().Err(err).Msg("failed to marshalIndent json") + // } + // deetsDigest := strings.Split(*detail.ImageDigest, ":")[1] + _, deetsDigest, _ := strings.Cut(*detail.ImageDigest, ":") + // fmt.Printf("deetsDigest: %v\n", deetsDigest) + // fmt.Printf("image.Digest: %v\n", image.Digest) + if deetsDigest == image.Digest { + + // fmt.Printf("running image %v:%v pushed at %v\n", *deets.RepositoryName, deets.ImageTags, deets.ImagePushedAt) + return false + } + } + + return true +} + +func GetPrometheusImagesFromProfile() ([]*ImageInfo, error) { + allImages := make([]*ImageInfo, 0) + datasource := Datasource{ + UID: "W7Xb02Snk", + } + query := Queries{ + RefID: "A", + Expr: "kube_pod_container_info", + + Datasource: datasource, + } + postBody := PostBody{ + From: "now-1h", + Queries: []Queries{query}, + To: "now", + } + buf, err := json.Marshal(postBody) + if err != nil { + log.Error().Err(err).Msg("failed to marshall json") + } + grafanaApiKey := os.Getenv("GRAFANA_API_KEY") + if grafanaApiKey == "" { + err = errors.New("env var GRAFANA_API_KEY not set") + log.Error().Err(err).Msg("failed to load environment variable") + return nil, err + } + req, err := http.NewRequest("POST", "https://grafana.ltvops.com/api/ds/query", bytes.NewBuffer(buf)) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", grafanaApiKey)) + req.Header.Add("Content-Type", "application/json") + if err != nil { + log.Error().Err(err).Msg("failed to build request") + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Error().Err(err).Msg("failed to call grafana") + return nil, err + } + if resp.StatusCode != 200 { + err = errors.New("non-200 status code") + log.Error().Err(err).Int("statusCode", resp.StatusCode).Str("status", resp.Status).Msg("failed to get a 200 on prometheus query") + return nil, err + } + defer resp.Body.Close() + + body := &Response{} + json.NewDecoder(resp.Body).Decode(body) + for _, item := range body.Results.A.Frames { + + for _, field := range item.Schema.Fields { + if field.Labels.Name == "kube_pod_container_info" { + registryUrl := strings.Split(strings.Split(field.Labels.Image, "@")[0], ":")[0] + if field.Labels.ImageID != "" { + digest := strings.Split(strings.Split(field.Labels.ImageID, "@")[1], ":")[1] + newImage := ImageInfo{ + FullImagePath: field.Labels.Image, + RegistryUrl: registryUrl, + RepositoryName: strings.Split(registryUrl, "/")[len(strings.Split(registryUrl, "/"))-1], + DeployedTag: strings.Split(field.Labels.Image, ":")[1], + Digest: digest, + Cluster: field.Labels.Cluster, + } + allImages = append(allImages, &newImage) + } + + } + } + } + return allImages, nil +} + +func GetECRImages(client *ecr.Client) ([]*ecr.DescribeImagesOutput, error) { + var allECRImages []*ecr.DescribeImagesOutput + registry, err := client.DescribeRegistry(context.Background(), &ecr.DescribeRegistryInput{}) + if err != nil { + log.Error().Err(err).Msg("failed to describe registry") + return nil, err + } + fmt.Printf("repos.RegistryId: %v\n", *registry.RegistryId) + repos, err := client.DescribeRepositories(context.Background(), &ecr.DescribeRepositoriesInput{}) + if err != nil { + log.Error().Err(err).Msg("failed to describe repositories") + return nil, err + } + for _, repo := range repos.Repositories { + if repo.RepositoryName != nil { + + input := &ecr.ListImagesInput{ + RepositoryName: repo.RepositoryName, + } + + images, err := client.ListImages(context.Background(), input) + if err != nil { + log.Error().Err(err).Msg("failed to list images") + return nil, err + } + if len(images.ImageIds) > 0 { + + describe, err := client.DescribeImages(context.Background(), &ecr.DescribeImagesInput{ + ImageIds: images.ImageIds, + RepositoryName: repo.RepositoryName, + }) + if err != nil { + log.Error().Err(err).Msg("failed to describe images") + return nil, err + } + allECRImages = append(allECRImages, describe) + + } + + } + } + return allECRImages, nil +} +func GetECRImage(ctx context.Context, client *ecr.Client, image types.ImageDetail) error { + data, err := client.BatchGetImage(ctx, &ecr.BatchGetImageInput{ + ImageIds: []types.ImageIdentifier{{ + ImageDigest: image.ImageDigest, + }}, + RepositoryName: image.RepositoryName, + RegistryId: image.RegistryId, + }) + if err != nil { + log.Error().Err(err).Msg("failed to get image") + return err + } + // delete, err := client.BatchDeleteImage(ctx, &ecr.BatchDeleteImageInput{ + // ImageIds: []types.ImageIdentifier{{ + // ImageDigest: image.ImageDigest, + // }}, + // RepositoryName: image.RepositoryName, + // RegistryId: image.RegistryId, + // }) + // if err != nil { + // log.Error().Err(err).Str("repoName", *image.RepositoryName).Str("imageDigest", *image.ImageDigest).Msg("failed to delete image") + // } + // for _, ii := range delete.ImageIds { + // fmt.Printf("ii.ImageDigest: %v\n", ii.ImageDigest) + // } + for _, i2 := range data.Images { + + if i2.ImageId.ImageTag != nil { + // fmt.Printf("RepoName:ImageDigest and tags: %v:%v %v\n", *i2.RepositoryName, *i2.ImageId.ImageDigest, *i2.ImageId.ImageTag) + continue + } + // fmt.Printf("RepoName:Digest: %v:%v\n", *i2.RepositoryName, *i2.ImageId.ImageDigest) + } + return nil +} + +func DeleteECRImage(ctx context.Context, client *ecr.Client, image types.ImageDetail) error { + delete, err := client.BatchDeleteImage(ctx, &ecr.BatchDeleteImageInput{ + ImageIds: []types.ImageIdentifier{{ + ImageDigest: image.ImageDigest, + }}, + RepositoryName: image.RepositoryName, + RegistryId: image.RegistryId, + }) + if err != nil { + log.Error().Err(err).Str("repoName", *image.RepositoryName).Str("imageDigest", *image.ImageDigest).Msg("failed to delete image") + return err + } + for _, ii := range delete.ImageIds { + fmt.Printf("ImageDigest and tags deleted: %v,\n", ii.ImageDigest) + } + // for _, i2 := range data.Images { + // fmt.Printf("i2.RepositoryName: %v\n", *i2.RepositoryName) + // if i2.ImageId.ImageTag != nil { + // fmt.Printf("i2.ImageId.ImageTag: %v\n", *i2.ImageId.ImageTag) + // } + // } + return nil +} + +func GetUnique(all []*ImageInfo) []*ImageInfo { + var unique []*ImageInfo + for _, v := range all { + skip := false + for _, u := range unique { + if v.FullImagePath == u.FullImagePath { + skip = true + break + } + } + if !skip { + unique = append(unique, v) + } + } + return unique +} + +func CleanRepos(untaggedOnly bool, keepLastCount int, profile string, region string, dryRun bool) bool { + // fmt.Printf("Cleaning %v profile of following images: \n\tUntagged Only: %v\n\tKeeping Last: %v\n\tDry Run: %v\n", profile, untaggedOnly, keepLastCount, dryRun) + log.Info().Bool("untaggedOnly", untaggedOnly).Bool("dryRun", dryRun).Int("keepLastCount", keepLastCount).Str("profile", profile).Msg("starting ecr-cleanup process") + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region), config.WithSharedConfigProfile(profile)) + if err != nil { + log.Fatal().Err(err).Msg("unable to load SDK config") + } + client := ecr.NewFromConfig(cfg) + + allImages, err := GetPrometheusImagesFromProfile() + if err != nil { + log.Error().Err(err).Msg("failed to get prometheus images") + return false + } + + allEcrImages, err := GetECRImages(client) + if err != nil { + log.Fatal().Err(err).Msg("failed to get ECR images") + return true + } + log.Debug().Int("prometheusImageCount", len(allImages)).Msg("number of prometheus images before deduplication") + // fmt.Printf("before unique allImages: %v\n", len(allImages)) + allImages = GetUnique(allImages) + log.Debug().Int("prometheusImageCount", len(allImages)).Msg("number of prometheus images after deduplication") + // fmt.Printf("after unique allImages: %v\n", len(allImages)) + var removeUntagged []*ecr.DescribeImagesOutput + var keepers []*ecr.DescribeImagesOutput + var cantDelete []*ecr.DescribeImagesOutput + for _, output := range allEcrImages { + untagged := new(ecr.DescribeImagesOutput) + keeper := new(ecr.DescribeImagesOutput) + sort.Slice(output.ImageDetails, func(i, j int) bool { + return output.ImageDetails[i].ImagePushedAt.After(*output.ImageDetails[j].ImagePushedAt) + }) + for _, deets := range output.ImageDetails { + if deets.ImageTags == nil { + untagged.ImageDetails = append(untagged.ImageDetails, deets) + continue + } + // fmt.Printf("deets.ImagePushedAt: %v: %v\n", *deets.RepositoryName, *deets.ImagePushedAt) + if !untaggedOnly { + if len(keeper.ImageDetails) <= keepLastCount { + keeper.ImageDetails = append(keeper.ImageDetails, deets) + continue + } + untagged.ImageDetails = append(untagged.ImageDetails, deets) + } + + } + keepers = append(keepers, keeper) + removeUntagged = append(removeUntagged, untagged) + + } + for _, unt := range removeUntagged { + noDelete := new(ecr.DescribeImagesOutput) + for _, deet := range unt.ImageDetails { + if deet.ImageDigest != nil { + if CheckImageNotInUse(allImages, deet) { + if dryRun { + GetECRImage(context.Background(), client, deet) + continue + } + DeleteECRImage(context.Background(), client, deet) + } else { + fmt.Printf("*****Can't delete %v: %v because it is in use!!!!\n", *deet.RepositoryName, *deet.ImageDigest) + log.Warn().Interface("details of image", deet).Msg("can't delete because image is in use") + noDelete.ImageDetails = append(noDelete.ImageDetails, deet) + + } + } + } + if noDelete.ImageDetails != nil { + cantDelete = append(cantDelete, noDelete) + } + } + // for _, k := range keepers { + // for _, v := range k.ImageDetails { + // // fmt.Printf("keeper.RepositoryName: %v:%v Tags:%v Pushed At: %v \n", *v.RepositoryName, *v.ImageDigest, v.ImageTags, v.ImagePushedAt) + // } + // } + + fmt.Printf("number of repos: %v\n", len(keepers)) + count := 0 + count2 := 0 + for _, dio := range keepers { + count += len(dio.ImageDetails) + + } + fmt.Printf("keeper images count: %v\n", count) + for _, dio := range removeUntagged { + count2 += len(dio.ImageDetails) + } + fmt.Printf("removeUntagged images count: %v\n", count2) + fmt.Printf("cantDelete: %v\n", len(cantDelete)) + for _, k := range cantDelete { + for _, v := range k.ImageDetails { + fmt.Printf("Currently in use: %v:%v Tags:%v Pushed At: %v \n", *v.RepositoryName, *v.ImageDigest, v.ImageTags, v.ImagePushedAt) + } + } + return false +} diff --git a/clean/models.go b/clean/models.go new file mode 100644 index 0000000..32f0417 --- /dev/null +++ b/clean/models.go @@ -0,0 +1,82 @@ +package clean + +import "time" + +type ImageInfo struct { + LastPushed time.Time + LastSeen time.Time + Digest string + RegistryUrl string + RepositoryName string + DeployedTag string + Tags []string + Cluster string + FullImagePath string +} + +type PostBody struct { + From string `json:"from"` + Queries []Queries `json:"queries"` + To string `json:"to"` +} +type Datasource struct { + UID string `json:"uid"` +} +type Queries struct { + Datasource Datasource `json:"datasource"` + Format string `json:"format"` + IntervalMs int `json:"intervalMs"` + MaxDataPoints int `json:"maxDataPoints"` + Expr string `json:"Expr"` + RefID string `json:"refId"` +} + +type Response struct { + Results struct { + A struct { + Status int `json:"status"` + Frames []struct { + Schema struct { + RefID string `json:"refId"` + Meta struct { + Type string `json:"type"` + TypeVersion []int `json:"typeVersion"` + Custom struct { + ResultType string `json:"resultType"` + } `json:"custom"` + ExecutedQueryString string `json:"executedQueryString"` + } `json:"meta"` + Fields []struct { + Name string `json:"name"` + Type string `json:"type"` + TypeInfo struct { + Frame string `json:"frame"` + } `json:"typeInfo"` + Config struct { + Interval int `json:"interval"` + } `json:"config,omitempty"` + Labels struct { + Name string `json:"__name__"` + Cluster string `json:"cluster"` + Container string `json:"container"` + ContainerID string `json:"container_id"` + Endpoint string `json:"endpoint"` + Image string `json:"image"` + ImageID string `json:"image_id"` + ImageSpec string `json:"image_spec"` + Instance string `json:"instance"` + Job string `json:"job"` + Namespace string `json:"namespace"` + Pod string `json:"pod"` + Service string `json:"service"` + UID string `json:"uid"` + } `json:"labels,omitempty"` + } `json:"fields"` + } `json:"schema"` + Data struct { + Values [][]int64 `json:"values"` + } `json:"data"` + } `json:"frames"` + } `json:"A"` + } `json:"results"` +} diff --git a/cmd/root.go b/cmd/root.go index 89d5f95..92b3519 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,47 +1,41 @@ package cmd import ( - "errors" "fmt" "os" - "github.com/sethpollack/ecr-cleaner/ecr" + "github.com/ltvco/ecr-cleaner/clean" "github.com/spf13/cobra" ) var ( - opts ecr.Opts - - runExample = ` - ecr-cleaner \ - -r "(^[0-9a-f]+$|-[0-9a-f]+$)" \ - -n foobar - ` - rootCmd = &cobra.Command{ - Use: "ecr-cleaner", - Short: "Clean untagged images from ECR", - Long: runExample, - PreRunE: func(cmd *cobra.Command, args []string) error { - if !opts.RunAll && opts.RepoName == "" { - return errors.New("You must specify either --repo-name or --all.") - } else { - return nil - } - }, - RunE: func(cmd *cobra.Command, args []string) error { - return ecr.CleanRepos(opts) - }, - } + untaggedOnly bool + dryRun bool + keepLast int + profileName string + region string ) func init() { - rootCmd.Flags().StringVarP(&opts.TagRegex, "tag-regex", "r", "(^[0-9a-f]+$|-[0-9a-f]+$)", "Image tag regex") - rootCmd.Flags().StringVarP(&opts.RepoName, "repo-name", "n", "", "Name of repo to clean.") - rootCmd.Flags().BoolVarP(&opts.RunAll, "all", "a", false, "Clean all repos.") - rootCmd.Flags().BoolVarP(&opts.DryRun, "dry-run", "d", true, "dry run to see tags that will be deleted.") + + rootCmd.PersistentFlags().BoolVarP(&untaggedOnly, "untagged-only", "u", true, "Only delete untagged images (default: true)") + rootCmd.PersistentFlags().IntVarP(&keepLast, "keep-latest-count", "k", 10, "Keep the latest X images (default: 10)") + rootCmd.PersistentFlags().StringVarP(&profileName, "profile-name", "p", "NA", "The AWS profile to use (default: '')") + rootCmd.PersistentFlags().StringVarP(®ion, "region", "r", "us-east-1", "AWS region (default: us-east-1)") + rootCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "d", true, "Dry run of the clean action (default: true)") + +} + +var rootCmd = &cobra.Command{ + Use: "ecr-cleaner", + Short: "clean amazon elastic container registries", + Run: func(cmd *cobra.Command, args []string) { + clean.CleanRepos(untaggedOnly, keepLast, profileName, region, dryRun) + }, } func Execute() { + if err := rootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) diff --git a/ecr/ecr.go b/ecr/ecr.go deleted file mode 100644 index b61dbc9..0000000 --- a/ecr/ecr.go +++ /dev/null @@ -1,163 +0,0 @@ -package ecr - -import ( - "log" - "net/http" - "regexp" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ecr" -) - -var service *ecr.ECR - -func ecrClient() *ecr.ECR { - if service == nil { - config := aws.NewConfig() - timeout := 500 * time.Millisecond - config = config.WithHTTPClient(&http.Client{Timeout: timeout}) - service = ecr.New(session.New(config)) - } - - return service -} - -func cleanRepo(repoName, tagRegex string, dryRun bool) error { - images, err := getImages(repoName) - if err != nil { - return err - } - - remove := []*ecr.ImageIdentifier{} - - for _, image := range images { - untagged, err := isUntagged(image, tagRegex) - if err != nil { - return err - } - - if untagged { - if dryRun { - log.Printf("[DRY RUN] Removing (%s) -> %v", repoName, stringify(image.ImageTags)) - } else { - remove = append(remove, &ecr.ImageIdentifier{ImageDigest: image.ImageDigest}) - } - } - } - - if dryRun { - return nil - } - - return deleteImages(repoName, remove) -} - -func getRepos() ([]*ecr.Repository, error) { - svc := ecrClient() - - result, err := svc.DescribeRepositories(&ecr.DescribeRepositoriesInput{}) - if err != nil { - return nil, err - } - - return result.Repositories, nil -} - -func getImages(repo string) ([]*ecr.ImageDetail, error) { - svc := ecrClient() - - output := &ecr.DescribeImagesOutput{} - - err := svc.DescribeImagesPages(&ecr.DescribeImagesInput{ - RepositoryName: aws.String(repo), - }, - func(page *ecr.DescribeImagesOutput, lastPage bool) bool { - output.ImageDetails = append(output.ImageDetails, page.ImageDetails...) - return !lastPage - }, - ) - if err != nil { - return nil, err - } - - return output.ImageDetails, nil -} - -func deleteImages(repoName string, ids []*ecr.ImageIdentifier) error { - svc := ecrClient() - - if len(ids) == 0 { - log.Printf("No untagged images found for: %s", repoName) - return nil - } - - _, err := svc.BatchDeleteImage(&ecr.BatchDeleteImageInput{ - ImageIds: ids, - RepositoryName: aws.String(repoName), - }) - if err != nil { - return err - } - - log.Printf("Deleted untagged images for: %s", repoName) - - return nil -} - -func isUntagged(id *ecr.ImageDetail, regex string) (bool, error) { - if len(id.ImageTags) == 0 { - return true, nil - } else { - for _, s := range id.ImageTags { - matched, err := regexp.MatchString(regex, *s) - if err != nil { - return false, err - } - if !matched { - return false, nil - } - } - return true, nil - } - return false, nil -} - -type Opts struct { - RepoName string - TagRegex string - RunAll bool - DryRun bool -} - -func CleanRepos(opts Opts) error { - if opts.RunAll { - repos, err := getRepos() - if err != nil { - return err - } - - for _, repo := range repos { - err := cleanRepo(*repo.RepositoryName, opts.TagRegex, opts.DryRun) - if err != nil { - return err - } - } - } else { - err := cleanRepo(opts.RepoName, opts.TagRegex, opts.DryRun) - if err != nil { - return err - } - } - - return nil -} - -func stringify(ss []*string) []string { - res := []string{} - for _, s := range ss { - res = append(res, *s) - } - return res -} diff --git a/go.mod b/go.mod index 5cc3273..a3d230e 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,9 @@ module github.com/ltvco/ecr-cleaner go 1.21.4 require ( - github.com/aws/aws-sdk-go v1.48.7 - github.com/aws/aws-sdk-go-v2 v1.23.5 github.com/aws/aws-sdk-go-v2/config v1.25.12 github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3 github.com/rs/zerolog v1.31.0 - github.com/sethpollack/ecr-cleaner v0.0.0-20180422155703-773c673bc438 github.com/spf13/cobra v1.8.0 ) @@ -33,4 +30,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect ) -require github.com/jmespath/go-jmespath v0.4.0 // indirect +require ( + github.com/aws/aws-sdk-go-v2 v1.23.5 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect +) diff --git a/go.sum b/go.sum index 7503731..fd5a303 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/aws/aws-sdk-go v1.48.7 h1:gDcOhmkohlNk20j0uWpko5cLBbwSkB+xpkshQO45F7Y= -github.com/aws/aws-sdk-go v1.48.7/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.23.5 h1:xK6C4udTyDMd82RFvNkDQxtAd00xlzFUtX4fF2nMZyg= github.com/aws/aws-sdk-go-v2 v1.23.5/go.mod h1:t3szzKfP0NeRU27uBFczDivYJjsmSnqI8kIvKyWb9ds= github.com/aws/aws-sdk-go-v2/config v1.25.12 h1:mF4cMuNh/2G+d19nWnm1vJ/ak0qK6SbqF0KtSX9pxu0= @@ -54,8 +52,6 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sethpollack/ecr-cleaner v0.0.0-20180422155703-773c673bc438 h1:EIeAyisomPrBcw56k/u+KtQxH+LQR0jGhtMfoxXURMo= -github.com/sethpollack/ecr-cleaner v0.0.0-20180422155703-773c673bc438/go.mod h1:O84OMiYeGGrIBRWRQLHionF3n8iNQHhVTFlesPKUQH8= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/main.go b/main.go index 9fcee92..d8b466d 100644 --- a/main.go +++ b/main.go @@ -1,289 +1,64 @@ package main import ( - "bytes" - "context" - "encoding/json" - "fmt" - "net/http" - "os" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/ecr" // agent "github.com/ltvco/ltv-apm-modules-go/agent" + "os" + "strconv" + + "github.com/ltvco/ecr-cleaner/cmd" "github.com/rs/zerolog" - "github.com/rs/zerolog/log" ) -type ImageInfo struct { - LastPushed time.Time - LastSeen time.Time - Digest string - RegistryUrl string - RepositoryName string - DeployedTag string - Tags []string - Cluster string - FullImagePath string -} - -type PostBody struct { - From string `json:"from"` - Queries []Queries `json:"queries"` - To string `json:"to"` -} -type Datasource struct { - UID string `json:"uid"` -} -type Queries struct { - Datasource Datasource `json:"datasource"` - Format string `json:"format"` - IntervalMs int `json:"intervalMs"` - MaxDataPoints int `json:"maxDataPoints"` - Expr string `json:"Expr"` - RefID string `json:"refId"` -} - -type Response struct { - Results struct { - A struct { - Status int `json:"status"` - Frames []struct { - Schema struct { - RefID string `json:"refId"` - Meta struct { - Type string `json:"type"` - TypeVersion []int `json:"typeVersion"` - Custom struct { - ResultType string `json:"resultType"` - } `json:"custom"` - ExecutedQueryString string `json:"executedQueryString"` - } `json:"meta"` - Fields []struct { - Name string `json:"name"` - Type string `json:"type"` - TypeInfo struct { - Frame string `json:"frame"` - } `json:"typeInfo"` - Config struct { - Interval int `json:"interval"` - } `json:"config,omitempty"` - Labels struct { - Name string `json:"__name__"` - Cluster string `json:"cluster"` - Container string `json:"container"` - ContainerID string `json:"container_id"` - Endpoint string `json:"endpoint"` - Image string `json:"image"` - ImageID string `json:"image_id"` - ImageSpec string `json:"image_spec"` - Instance string `json:"instance"` - Job string `json:"job"` - Namespace string `json:"namespace"` - Pod string `json:"pod"` - Service string `json:"service"` - UID string `json:"uid"` - } `json:"labels,omitempty"` - } `json:"fields"` - } `json:"schema"` - Data struct { - Values [][]int64 `json:"values"` - } `json:"data"` - } `json:"frames"` - } `json:"A"` - } `json:"results"` -} - func main() { - zerolog.SetGlobalLevel(zerolog.InfoLevel) - - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion("us-east-1"), config.WithSharedConfigProfile("production")) - if err != nil { - log.Fatal().Err(err).Msg("unable to load SDK config") - } - - allImages := GetPrometheusImagesFromProfile() - - allEcrImages, err := GetECRImages(cfg) - if err != nil { - log.Fatal().Err(err).Msg("failed to get ECR images") - return - } - fmt.Printf("len(allImages): %v\n", len(allImages)) - for i, output := range allEcrImages { - untagged := 0 - for _, deets := range output.ImageDetails { - if deets.ImageTags == nil { - untagged++ - continue - } - // fmt.Printf("Index: %v imageName and tags: %v:%v\n", i, *deets.RepositoryName, deets.ImageTags) - // for _, image := range allImages { - - // if *deets.RepositoryName == image.RepositoryName { - // s, _ := json.MarshalIndent(deets, "", "\t") - // fmt.Printf("image.RepositoryName: %v\n", image.RepositoryName) - // fmt.Printf("s: %v\n", string(s)) - - // image.Tags = deets.ImageTags - // image.LastPushed = *deets.ImagePushedAt - // image.Digest = *deets.ImageDigest - - // } - // } - } - fmt.Printf("untagged in %v: %v\n", i, untagged) - } - fmt.Printf("before unique allImages: %v\n", len(allImages)) - allImages = GetUnique(allImages) - fmt.Printf("after unique allImages: %v\n", len(allImages)) - - for _, image := range allImages { - _, err := json.MarshalIndent(image, "", "\t") - if err != nil { - log.Error().Err(err).Msg("failed to marshalIndent json") - } - // fmt.Printf("s: %v\n", string(s)) - - // if image.LastPushed.Before(time.Now().AddDate(0, 0, -(daysOld))) { - // fmt.Printf("%v is older than %v days old\n", image.FullImagePath, daysOld) - // } - for _, ecr := range allEcrImages { - for _, deets := range ecr.ImageDetails { - // fmt.Printf("deets.ImageDigest: %v\n", *deets.ImageDigest) - deetsDigest := strings.Split(*deets.ImageDigest, ":")[1] - // fmt.Printf("deetsDigest: %v\n", deetsDigest) - // fmt.Printf("image.Digest: %v\n", image.Digest) - if deetsDigest == image.Digest { - - fmt.Printf("running image %v:%v pushed at %v\n", *deets.RepositoryName, deets.ImageTags, deets.ImagePushedAt) - continue - } - } - } - } - -} -func GetPrometheusImagesFromProfile() []*ImageInfo { - allImages := make([]*ImageInfo, 0) - datasource := Datasource{ - UID: "W7Xb02Snk", - } - query := Queries{ - RefID: "A", - Expr: "kube_pod_container_info{cluster=\"promoted-walleye\"}", - - Datasource: datasource, - } - postBody := PostBody{ - From: "now-1h", - Queries: []Queries{query}, - To: "now", - } - buf, err := json.Marshal(postBody) + logLevel, err := strconv.Atoi(os.Getenv("LOG_LEVEL")) if err != nil { - log.Error().Err(err).Msg("failed to marshall json") - } - grafanaApiKey := os.Getenv("GRAFANA_API_KEY") - req, _ := http.NewRequest("POST", "https://grafana.ltvops.com/api/ds/query", bytes.NewBuffer(buf)) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", grafanaApiKey)) - req.Header.Add("Content-Type", "application/json") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - log.Error().Err(err).Msg("failed to call grafana") - } - defer resp.Body.Close() - - body := &Response{} - json.NewDecoder(resp.Body).Decode(body) - for _, item := range body.Results.A.Frames { - - for _, field := range item.Schema.Fields { - if field.Labels.Name == "kube_pod_container_info" { - - // var lastSeen time.Time - // if item.Data.Values[1][0] != 0 { - // lastSeen = time.Unix(item.Data.Values[1][0], 0) - // } - - registryUrl := strings.Split(strings.Split(field.Labels.Image, "@")[0], ":")[0] - if field.Labels.ImageID != "" { - // fmt.Printf("field.Labels.ImageID: %v\n", field.Labels.ImageID) - digest := strings.Split(strings.Split(field.Labels.ImageID, "@")[1], ":")[1] - newImage := ImageInfo{ - FullImagePath: field.Labels.Image, - RegistryUrl: registryUrl, - RepositoryName: strings.Split(registryUrl, "/")[len(strings.Split(registryUrl, "/"))-1], - DeployedTag: strings.Split(field.Labels.Image, ":")[1], - Digest: digest, - Cluster: field.Labels.Cluster, - } - allImages = append(allImages, &newImage) - } - - } - } + logLevel = int(zerolog.InfoLevel) // default to INFO } - return allImages -} - -func GetECRImages(cfg aws.Config) ([]*ecr.DescribeImagesOutput, error) { - var allECRImages []*ecr.DescribeImagesOutput - client := ecr.NewFromConfig(cfg) - registry, _ := client.DescribeRegistry(context.Background(), &ecr.DescribeRegistryInput{}) - fmt.Printf("repos.RegistryId: %v\n", *registry.RegistryId) - repos, _ := client.DescribeRepositories(context.Background(), &ecr.DescribeRepositoriesInput{}) - for _, repo := range repos.Repositories { - if repo.RepositoryName != nil { - - input := &ecr.ListImagesInput{ - RepositoryName: repo.RepositoryName, - } - - images, err := client.ListImages(context.Background(), input) - if err != nil { - log.Error().Err(err).Msg("failed to list images") - return nil, err - } - if len(images.ImageIds) > 0 { + zerolog.SetGlobalLevel(zerolog.Level(logLevel)) + + cmd.Execute() + // fmt.Printf("Index: %v imageName and tags: %v:%v\n", i, *deets.RepositoryName, deets.ImageTags) + // for _, image := range allImages { + // if *deets.RepositoryName == image.RepositoryName { + // s, _ := json.MarshalIndent(deets, "", "\t") + // fmt.Printf("image.RepositoryName: %v\n", image.RepositoryName) + // fmt.Printf("s: %v\n", string(s)) + // image.Tags = deets.ImageTags + // image.LastPushed = *deets.ImagePushedAt + // image.Digest = *deets.ImageDigest + // } + // } + // fmt.Printf("untagged in %v: %v\n", i, untagged.ImageDetails) + // fmt.Printf("deet.RepositoryName: %v:%v Tags: %v\n", *deet.RepositoryName, *deet.ImageDigest, deet.ImageTags) + // fmt.Printf("unt.ImageDetails: %v\n", unt.ImageDetails) + // fmt.Printf("removeUntagged: %v\n", removeUntagged) + + // for _, image := range allImages { + // _, err := json.MarshalIndent(image, "", "\t") + // if err != nil { + // log.Error().Err(err).Msg("failed to marshalIndent json") + // } + // // fmt.Printf("s: %v\n", string(s)) + + // // if image.LastPushed.Before(time.Now().AddDate(0, 0, -(daysOld))) { + // // fmt.Printf("%v is older than %v days old\n", image.FullImagePath, daysOld) + // // } + // for _, ecr := range keepers { + // for _, deets := range ecr.ImageDetails { + // // fmt.Printf("deets.ImageDigest: %v\n", *deets.ImageDigest) + // deetsDigest := strings.Split(*deets.ImageDigest, ":")[1] + // // fmt.Printf("deetsDigest: %v\n", deetsDigest) + // // fmt.Printf("image.Digest: %v\n", image.Digest) + // if deetsDigest == image.Digest { + + // // fmt.Printf("running image %v:%v pushed at %v\n", *deets.RepositoryName, deets.ImageTags, deets.ImagePushedAt) + // continue + // } + // } + // } + // } - describe, err := client.DescribeImages(context.Background(), &ecr.DescribeImagesInput{ - ImageIds: images.ImageIds, - RepositoryName: repo.RepositoryName, - }) - if err != nil { - log.Error().Err(err).Msg("failed to describe images") - return nil, err - } - allECRImages = append(allECRImages, describe) - - } - - } - } - return allECRImages, nil -} - -func GetUnique(all []*ImageInfo) []*ImageInfo { - var unique []*ImageInfo - for _, v := range all { - skip := false - for _, u := range unique { - if v.FullImagePath == u.FullImagePath { - skip = true - break - } - } - if !skip { - unique = append(unique, v) - } - } - return unique } From cb09ff33341a40547f64ed6f55a3f0a9695952f6 Mon Sep 17 00:00:00 2001 From: Will Lamm Date: Thu, 11 Jan 2024 14:16:44 -0600 Subject: [PATCH 3/4] fix: assuming roles instead of profiles --- Dockerfile | 26 +++++----- clean/clean.go | 131 ++++++++++++++++++++++++++---------------------- clean/models.go | 18 +++---- cmd/root.go | 4 +- go.mod | 4 +- main.go | 42 ---------------- 6 files changed, 97 insertions(+), 128 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6b24cf1..511e2a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,17 @@ -FROM golang:1.9-alpine AS build -RUN apk --no-cache update && \ - apk --no-cache add make ca-certificates git && \ - rm -rf /var/cache/apk/* -WORKDIR /go/src/github.com/sethpollack/ecr-cleaner -RUN go get -u github.com/golang/dep/cmd/dep -COPY Gopkg.toml Gopkg.lock ./ -RUN dep ensure -v -vendor-only +FROM golang:1.21-alpine AS builder +WORKDIR / +RUN apk --no-cache add ca-certificates git && rm -rf /var/cache/apk/* +ENV GO111MODULE=auto COPY . ./ -RUN CGO_ENABLED=0 GOOS=linux go build -installsuffix cgo -o bin/ecr-cleaner +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o ecr-cleaner -FROM scratch -LABEL maintainer="Seth Pollack " -COPY --from=build /etc/ssl/certs/ /etc/ssl/certs/ -COPY --from=build /go/src/github.com/sethpollack/ecr-cleaner/bin/ecr-cleaner /usr/local/bin/ecr-cleaner +FROM alpine:3 +RUN apk --no-cache add ca-certificates && rm -rf /var/cache/apk/* +WORKDIR /usr/local/bin/ +RUN adduser -D -H -s /usr/sbin/nologin app +USER app +# RUN mkdir /home/app +COPY --from=builder ecr-cleaner ./ +COPY aws/ /home/app/.aws/ ENTRYPOINT ["/usr/local/bin/ecr-cleaner"] CMD ["--help"] diff --git a/clean/clean.go b/clean/clean.go index 1ae272f..7ee9f84 100644 --- a/clean/clean.go +++ b/clean/clean.go @@ -6,34 +6,54 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "os" "sort" "strings" + "time" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" "github.com/aws/aws-sdk-go-v2/service/ecr" "github.com/aws/aws-sdk-go-v2/service/ecr/types" - "github.com/rs/zerolog/log" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/rs/zerolog" ) +var log zerolog.Logger + +func init() { + var output io.Writer = zerolog.ConsoleWriter{ + Out: os.Stderr, + TimeFormat: time.RFC3339, + FormatLevel: func(i interface{}) string { + return strings.ToUpper(fmt.Sprintf("[%s]", i)) + }, + FormatMessage: func(i interface{}) string { + return fmt.Sprintf("| %s |", i) + }, + // FormatCaller: func(i interface{}) string { + // return filepath.Base(fmt.Sprintf("%s", i)) + // }, + PartsExclude: []string{ + zerolog.TimestampFieldName, + }, + } + if os.Getenv("GO_ENV") != "development" { + output = os.Stderr + } + log = zerolog.New(output) +} + func CheckImageNotInUse(all []*ImageInfo, detail types.ImageDetail) bool { for _, image := range all { - // _, err := json.MarshalIndent(image, "", "\t") - // if err != nil { - // log.Error().Err(err).Msg("failed to marshalIndent json") - // } - // deetsDigest := strings.Split(*detail.ImageDigest, ":")[1] _, deetsDigest, _ := strings.Cut(*detail.ImageDigest, ":") - // fmt.Printf("deetsDigest: %v\n", deetsDigest) - // fmt.Printf("image.Digest: %v\n", image.Digest) if deetsDigest == image.Digest { - - // fmt.Printf("running image %v:%v pushed at %v\n", *deets.RepositoryName, deets.ImageTags, deets.ImagePushedAt) return false } } - return true } @@ -85,7 +105,6 @@ func GetPrometheusImagesFromProfile() ([]*ImageInfo, error) { body := &Response{} json.NewDecoder(resp.Body).Decode(body) for _, item := range body.Results.A.Frames { - for _, field := range item.Schema.Fields { if field.Labels.Name == "kube_pod_container_info" { registryUrl := strings.Split(strings.Split(field.Labels.Image, "@")[0], ":")[0] @@ -101,7 +120,6 @@ func GetPrometheusImagesFromProfile() ([]*ImageInfo, error) { } allImages = append(allImages, &newImage) } - } } } @@ -115,7 +133,8 @@ func GetECRImages(client *ecr.Client) ([]*ecr.DescribeImagesOutput, error) { log.Error().Err(err).Msg("failed to describe registry") return nil, err } - fmt.Printf("repos.RegistryId: %v\n", *registry.RegistryId) + log.Debug().Str("awsAccount", *registry.RegistryId).Msg("AWS Account number to be scanned") + // fmt.Printf("repos.RegistryId: %v\n", *registry.RegistryId) repos, err := client.DescribeRepositories(context.Background(), &ecr.DescribeRepositoriesInput{}) if err != nil { log.Error().Err(err).Msg("failed to describe repositories") @@ -123,18 +142,15 @@ func GetECRImages(client *ecr.Client) ([]*ecr.DescribeImagesOutput, error) { } for _, repo := range repos.Repositories { if repo.RepositoryName != nil { - input := &ecr.ListImagesInput{ RepositoryName: repo.RepositoryName, } - images, err := client.ListImages(context.Background(), input) if err != nil { log.Error().Err(err).Msg("failed to list images") return nil, err } if len(images.ImageIds) > 0 { - describe, err := client.DescribeImages(context.Background(), &ecr.DescribeImagesInput{ ImageIds: images.ImageIds, RepositoryName: repo.RepositoryName, @@ -144,14 +160,14 @@ func GetECRImages(client *ecr.Client) ([]*ecr.DescribeImagesOutput, error) { return nil, err } allECRImages = append(allECRImages, describe) - } - } } return allECRImages, nil } + func GetECRImage(ctx context.Context, client *ecr.Client, image types.ImageDetail) error { + log.Debug().Str("imageRepositoryName", *image.RepositoryName).Msg("getting image from ECR") data, err := client.BatchGetImage(ctx, &ecr.BatchGetImageInput{ ImageIds: []types.ImageIdentifier{{ ImageDigest: image.ImageDigest, @@ -163,31 +179,17 @@ func GetECRImage(ctx context.Context, client *ecr.Client, image types.ImageDetai log.Error().Err(err).Msg("failed to get image") return err } - // delete, err := client.BatchDeleteImage(ctx, &ecr.BatchDeleteImageInput{ - // ImageIds: []types.ImageIdentifier{{ - // ImageDigest: image.ImageDigest, - // }}, - // RepositoryName: image.RepositoryName, - // RegistryId: image.RegistryId, - // }) - // if err != nil { - // log.Error().Err(err).Str("repoName", *image.RepositoryName).Str("imageDigest", *image.ImageDigest).Msg("failed to delete image") - // } - // for _, ii := range delete.ImageIds { - // fmt.Printf("ii.ImageDigest: %v\n", ii.ImageDigest) - // } - for _, i2 := range data.Images { - - if i2.ImageId.ImageTag != nil { - // fmt.Printf("RepoName:ImageDigest and tags: %v:%v %v\n", *i2.RepositoryName, *i2.ImageId.ImageDigest, *i2.ImageId.ImageTag) - continue - } - // fmt.Printf("RepoName:Digest: %v:%v\n", *i2.RepositoryName, *i2.ImageId.ImageDigest) + for _, v := range data.Images { + log.Info().Interface("imageDetails", v).Msg("GetECRImage details") } return nil } -func DeleteECRImage(ctx context.Context, client *ecr.Client, image types.ImageDetail) error { +func DeleteECRImage(ctx context.Context, client *ecr.Client, image types.ImageDetail, dryRun bool) error { + if dryRun { + log.Info().Interface("imageDetail", image).Msg("dry run of delete image") + return nil + } delete, err := client.BatchDeleteImage(ctx, &ecr.BatchDeleteImageInput{ ImageIds: []types.ImageIdentifier{{ ImageDigest: image.ImageDigest, @@ -200,14 +202,9 @@ func DeleteECRImage(ctx context.Context, client *ecr.Client, image types.ImageDe return err } for _, ii := range delete.ImageIds { - fmt.Printf("ImageDigest and tags deleted: %v,\n", ii.ImageDigest) + log.Info().Interface("deletedImage", ii).Msg("deleted image") + // fmt.Printf("ImageDigest and tags deleted: %v,\n", ii.ImageDigest) } - // for _, i2 := range data.Images { - // fmt.Printf("i2.RepositoryName: %v\n", *i2.RepositoryName) - // if i2.ImageId.ImageTag != nil { - // fmt.Printf("i2.ImageId.ImageTag: %v\n", *i2.ImageId.ImageTag) - // } - // } return nil } @@ -228,15 +225,21 @@ func GetUnique(all []*ImageInfo) []*ImageInfo { return unique } -func CleanRepos(untaggedOnly bool, keepLastCount int, profile string, region string, dryRun bool) bool { - // fmt.Printf("Cleaning %v profile of following images: \n\tUntagged Only: %v\n\tKeeping Last: %v\n\tDry Run: %v\n", profile, untaggedOnly, keepLastCount, dryRun) - log.Info().Bool("untaggedOnly", untaggedOnly).Bool("dryRun", dryRun).Int("keepLastCount", keepLastCount).Str("profile", profile).Msg("starting ecr-cleanup process") - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region), config.WithSharedConfigProfile(profile)) +func CleanRepos(untaggedOnly bool, keepLastCount int, profile string, region string, dryRun bool, verbose bool) bool { + arnmap := map[string]string{ + "development": "arn:aws:iam::722014088219:role/devops-read-only", + "production": "arn:aws:iam::667347940230:role/devops-read-only", + } + log.Info().Bool("untaggedOnly", untaggedOnly).Bool("dryRun", dryRun).Int("keepLastCount", keepLastCount).Str("profile", profile).Bool("verbose", verbose).Str("region", region).Msg("starting ecr-cleanup process") + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region)) if err != nil { log.Fatal().Err(err).Msg("unable to load SDK config") } - client := ecr.NewFromConfig(cfg) + stsSvc := sts.NewFromConfig(cfg) + stsCred := stscreds.NewAssumeRoleProvider(stsSvc, arnmap[profile]) + cfg.Credentials = aws.NewCredentialsCache(stsCred) + client := ecr.NewFromConfig(cfg) allImages, err := GetPrometheusImagesFromProfile() if err != nil { log.Error().Err(err).Msg("failed to get prometheus images") @@ -286,13 +289,12 @@ func CleanRepos(untaggedOnly bool, keepLastCount int, profile string, region str for _, deet := range unt.ImageDetails { if deet.ImageDigest != nil { if CheckImageNotInUse(allImages, deet) { - if dryRun { + DeleteECRImage(context.Background(), client, deet, dryRun) + if verbose { GetECRImage(context.Background(), client, deet) - continue } - DeleteECRImage(context.Background(), client, deet) } else { - fmt.Printf("*****Can't delete %v: %v because it is in use!!!!\n", *deet.RepositoryName, *deet.ImageDigest) + // fmt.Printf("*****Can't delete %v: %v because it is in use!!!!\n", *deet.RepositoryName, *deet.ImageDigest) log.Warn().Interface("details of image", deet).Msg("can't delete because image is in use") noDelete.ImageDetails = append(noDelete.ImageDetails, deet) @@ -309,22 +311,29 @@ func CleanRepos(untaggedOnly bool, keepLastCount int, profile string, region str // } // } - fmt.Printf("number of repos: %v\n", len(keepers)) + log.Debug().Int("numberRepos", len(keepers)).Msg("number of repositories scanned") + // fmt.Printf("number of repos: %v\n", len(keepers)) count := 0 count2 := 0 for _, dio := range keepers { count += len(dio.ImageDetails) } - fmt.Printf("keeper images count: %v\n", count) + log.Info().Int("keepImagesCount", count).Msg("number of images to keep") + // fmt.Printf("keeper images count: %v\n", count) for _, dio := range removeUntagged { count2 += len(dio.ImageDetails) } - fmt.Printf("removeUntagged images count: %v\n", count2) - fmt.Printf("cantDelete: %v\n", len(cantDelete)) + log.Info().Int("removedImagesCount", count2).Msg("total number of images removed") + // fmt.Printf("removeUntagged images count: %v\n", count2) + if len(cantDelete) > 0 { + log.Info().Int("cannotDelete", len(cantDelete)).Msg("number of images that cannot be deleted due to being in use") + } + // fmt.Printf("cantDelete: %v\n", len(cantDelete)) for _, k := range cantDelete { for _, v := range k.ImageDetails { - fmt.Printf("Currently in use: %v:%v Tags:%v Pushed At: %v \n", *v.RepositoryName, *v.ImageDigest, v.ImageTags, v.ImagePushedAt) + log.Info().Interface("cannotDeleteImage", v).Msg("image cannot be deleted") + // fmt.Printf("Currently in use: %v:%v Tags:%v Pushed At: %v \n", *v.RepositoryName, *v.ImageDigest, v.ImageTags, v.ImagePushedAt) } } return false diff --git a/clean/models.go b/clean/models.go index 32f0417..be4390d 100644 --- a/clean/models.go +++ b/clean/models.go @@ -3,15 +3,15 @@ package clean import "time" type ImageInfo struct { - LastPushed time.Time - LastSeen time.Time - Digest string - RegistryUrl string - RepositoryName string - DeployedTag string - Tags []string - Cluster string - FullImagePath string + LastPushed time.Time `json:"last_pushed,omitempty"` + LastSeen time.Time `json:"last_seen,omitempty"` + Digest string `json:"digest,omitempty"` + RegistryUrl string `json:"registry_url,omitempty"` + RepositoryName string `json:"repository_name,omitempty"` + DeployedTag string `json:"deployed_tag,omitempty"` + Tags []string `json:"tags,omitempty"` + Cluster string `json:"cluster,omitempty"` + FullImagePath string `json:"full_image_path,omitempty"` } type PostBody struct { diff --git a/cmd/root.go b/cmd/root.go index 92b3519..18da302 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,6 +11,7 @@ import ( var ( untaggedOnly bool dryRun bool + verbose bool keepLast int profileName string region string @@ -23,6 +24,7 @@ func init() { rootCmd.PersistentFlags().StringVarP(&profileName, "profile-name", "p", "NA", "The AWS profile to use (default: '')") rootCmd.PersistentFlags().StringVarP(®ion, "region", "r", "us-east-1", "AWS region (default: us-east-1)") rootCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "d", true, "Dry run of the clean action (default: true)") + rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Use to see details of images to be deleted") } @@ -30,7 +32,7 @@ var rootCmd = &cobra.Command{ Use: "ecr-cleaner", Short: "clean amazon elastic container registries", Run: func(cmd *cobra.Command, args []string) { - clean.CleanRepos(untaggedOnly, keepLast, profileName, region, dryRun) + clean.CleanRepos(untaggedOnly, keepLast, profileName, region, dryRun, verbose) }, } diff --git a/go.mod b/go.mod index a3d230e..1f156a3 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( ) require ( - github.com/aws/aws-sdk-go-v2/credentials v1.16.10 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.16.10 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.9 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.8 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.8 // indirect @@ -19,7 +19,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.8 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.18.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.26.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.3 github.com/aws/smithy-go v1.18.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/main.go b/main.go index d8b466d..5e92b6e 100644 --- a/main.go +++ b/main.go @@ -12,7 +12,6 @@ import ( ) func main() { - logLevel, err := strconv.Atoi(os.Getenv("LOG_LEVEL")) if err != nil { logLevel = int(zerolog.InfoLevel) // default to INFO @@ -20,45 +19,4 @@ func main() { zerolog.SetGlobalLevel(zerolog.Level(logLevel)) cmd.Execute() - // fmt.Printf("Index: %v imageName and tags: %v:%v\n", i, *deets.RepositoryName, deets.ImageTags) - // for _, image := range allImages { - // if *deets.RepositoryName == image.RepositoryName { - // s, _ := json.MarshalIndent(deets, "", "\t") - // fmt.Printf("image.RepositoryName: %v\n", image.RepositoryName) - // fmt.Printf("s: %v\n", string(s)) - // image.Tags = deets.ImageTags - // image.LastPushed = *deets.ImagePushedAt - // image.Digest = *deets.ImageDigest - // } - // } - // fmt.Printf("untagged in %v: %v\n", i, untagged.ImageDetails) - // fmt.Printf("deet.RepositoryName: %v:%v Tags: %v\n", *deet.RepositoryName, *deet.ImageDigest, deet.ImageTags) - // fmt.Printf("unt.ImageDetails: %v\n", unt.ImageDetails) - // fmt.Printf("removeUntagged: %v\n", removeUntagged) - - // for _, image := range allImages { - // _, err := json.MarshalIndent(image, "", "\t") - // if err != nil { - // log.Error().Err(err).Msg("failed to marshalIndent json") - // } - // // fmt.Printf("s: %v\n", string(s)) - - // // if image.LastPushed.Before(time.Now().AddDate(0, 0, -(daysOld))) { - // // fmt.Printf("%v is older than %v days old\n", image.FullImagePath, daysOld) - // // } - // for _, ecr := range keepers { - // for _, deets := range ecr.ImageDetails { - // // fmt.Printf("deets.ImageDigest: %v\n", *deets.ImageDigest) - // deetsDigest := strings.Split(*deets.ImageDigest, ":")[1] - // // fmt.Printf("deetsDigest: %v\n", deetsDigest) - // // fmt.Printf("image.Digest: %v\n", image.Digest) - // if deetsDigest == image.Digest { - - // // fmt.Printf("running image %v:%v pushed at %v\n", *deets.RepositoryName, deets.ImageTags, deets.ImagePushedAt) - // continue - // } - // } - // } - // } - } From 0e9a62c0728b51b429cdb709893b273adf7a6677 Mon Sep 17 00:00:00 2001 From: Will Lamm Date: Mon, 20 May 2024 10:50:01 -0500 Subject: [PATCH 4/4] added envvar and arm flag options --- clean/clean.go | 32 ++++++-- clean/clean_test.go | 181 ++++++++++++++++++++++++++++++++++++++++++++ cmd/root.go | 9 +-- go.mod | 8 +- go.sum | 3 +- 5 files changed, 216 insertions(+), 17 deletions(-) create mode 100644 clean/clean_test.go diff --git a/clean/clean.go b/clean/clean.go index 7ee9f84..cfa077c 100644 --- a/clean/clean.go +++ b/clean/clean.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "os" + "regexp" "sort" "strings" "time" @@ -51,6 +52,7 @@ func CheckImageNotInUse(all []*ImageInfo, detail types.ImageDetail) bool { for _, image := range all { _, deetsDigest, _ := strings.Cut(*detail.ImageDigest, ":") if deetsDigest == image.Digest { + return false } } @@ -225,18 +227,36 @@ func GetUnique(all []*ImageInfo) []*ImageInfo { return unique } -func CleanRepos(untaggedOnly bool, keepLastCount int, profile string, region string, dryRun bool, verbose bool) bool { - arnmap := map[string]string{ - "development": "arn:aws:iam::722014088219:role/devops-read-only", - "production": "arn:aws:iam::667347940230:role/devops-read-only", +func CleanRepos(untaggedOnly bool, keepLastCount int, arn string, region string, dryRun bool, verbose bool) bool { + // arnmap := map[string]string{ + // "development": "arn:aws:iam::722014088219:role/devops-read-only", + // "production": "arn:aws:iam::667347940230:role/devops-read-only", + //} + + if arn == "arn:aws:iam::722014088219:role/devops-read-only" { + if os.Getenv("AWS_ROLE_ARN") != "" { + arn = os.Getenv("AWS_ROLE_ARN") + } } - log.Info().Bool("untaggedOnly", untaggedOnly).Bool("dryRun", dryRun).Int("keepLastCount", keepLastCount).Str("profile", profile).Bool("verbose", verbose).Str("region", region).Msg("starting ecr-cleanup process") + fmt.Printf("arn: %v\n", arn) + match, err := regexp.MatchString("arn:aws:iam::\\d.+:\\w+\\/.+", arn) + if err != nil { + log.Err(err).Msg("failed to match ARN") + os.Exit(1) + } + if !match { + log.Error().Err(err).Msg("ARN does not match pattern") + os.Exit(1) + } + log.Info().Bool("untaggedOnly", untaggedOnly).Bool("dryRun", dryRun).Int("keepLastCount", keepLastCount).Str("arnRole", strings.Split(arn, ":")[len(strings.Split(arn, ":"))-1]).Bool("verbose", verbose).Str("region", region).Msg("starting ecr-cleanup process") cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(region)) if err != nil { log.Fatal().Err(err).Msg("unable to load SDK config") } stsSvc := sts.NewFromConfig(cfg) - stsCred := stscreds.NewAssumeRoleProvider(stsSvc, arnmap[profile]) + + // arn := arnmap[profile] + stsCred := stscreds.NewAssumeRoleProvider(stsSvc, arn) cfg.Credentials = aws.NewCredentialsCache(stsCred) client := ecr.NewFromConfig(cfg) diff --git a/clean/clean_test.go b/clean/clean_test.go new file mode 100644 index 0000000..0dd0c7e --- /dev/null +++ b/clean/clean_test.go @@ -0,0 +1,181 @@ +package clean + +import ( + "github.com/aws/aws-sdk-go-v2/service/ecr/types" + "testing" +) + +func TestCheckImageNotInUse(t *testing.T) { + type args struct { + all []*ImageInfo + detail types.ImageDetail + } + digest := "sha256:b31dd6ba7d28a1559be39a88c292a1a8948491b118dafd3e8139065afe55690a" + tests := []struct { + name string + args args + want bool + }{{ + name: "Match", + args: args{ + all: []*ImageInfo{{Digest: "b31dd6ba7d28a1559be39a88c292a1a8948491b118dafd3e8139065afe55690a"}}, + detail: types.ImageDetail{ImageDigest: &digest}, + }, + want: false, + }, + { + name: "NoMatch", + args: args{ + all: []*ImageInfo{{Digest: "13b7e62e8df80264dbb747995705a986aa530415763a6c58f84a3ca8af9a5bcd"}}, + detail: types.ImageDetail{ImageDigest: &digest}, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CheckImageNotInUse(tt.args.all, tt.args.detail); got != tt.want { + t.Errorf("CheckImageNotInUse() = %v, want %v", got, tt.want) + } + }) + } +} + +// +//func TestCleanRepos(t *testing.T) { +// type args struct { +// untaggedOnly bool +// keepLastCount int +// profile string +// region string +// dryRun bool +// verbose bool +// } +// tests := []struct { +// name string +// args args +// want bool +// }{ +// // TODO: Add test cases. +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// if got := CleanRepos(tt.args.untaggedOnly, tt.args.keepLastCount, tt.args.profile, tt.args.region, tt.args.dryRun, tt.args.verbose); got != tt.want { +// t.Errorf("CleanRepos() = %v, want %v", got, tt.want) +// } +// }) +// } +//} +// +//func TestDeleteECRImage(t *testing.T) { +// type args struct { +// ctx context.Context +// client *ecr.Client +// image types.ImageDetail +// dryRun bool +// } +// tests := []struct { +// name string +// args args +// wantErr bool +// }{ +// // TODO: Add test cases. +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// if err := DeleteECRImage(tt.args.ctx, tt.args.client, tt.args.image, tt.args.dryRun); (err != nil) != tt.wantErr { +// t.Errorf("DeleteECRImage() error = %v, wantErr %v", err, tt.wantErr) +// } +// }) +// } +//} +// +//func TestGetECRImage(t *testing.T) { +// type args struct { +// ctx context.Context +// client *ecr.Client +// image types.ImageDetail +// } +// tests := []struct { +// name string +// args args +// wantErr bool +// }{ +// // TODO: Add test cases. +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// if err := GetECRImage(tt.args.ctx, tt.args.client, tt.args.image); (err != nil) != tt.wantErr { +// t.Errorf("GetECRImage() error = %v, wantErr %v", err, tt.wantErr) +// } +// }) +// } +//} +// +//func TestGetECRImages(t *testing.T) { +// type args struct { +// client *ecr.Client +// } +// tests := []struct { +// name string +// args args +// want []*ecr.DescribeImagesOutput +// wantErr bool +// }{ +// // TODO: Add test cases. +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// got, err := GetECRImages(tt.args.client) +// if (err != nil) != tt.wantErr { +// t.Errorf("GetECRImages() error = %v, wantErr %v", err, tt.wantErr) +// return +// } +// if !reflect.DeepEqual(got, tt.want) { +// t.Errorf("GetECRImages() got = %v, want %v", got, tt.want) +// } +// }) +// } +//} +// +//func TestGetPrometheusImagesFromProfile(t *testing.T) { +// tests := []struct { +// name string +// want []*ImageInfo +// wantErr bool +// }{ +// // TODO: Add test cases. +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// got, err := GetPrometheusImagesFromProfile() +// if (err != nil) != tt.wantErr { +// t.Errorf("GetPrometheusImagesFromProfile() error = %v, wantErr %v", err, tt.wantErr) +// return +// } +// if !reflect.DeepEqual(got, tt.want) { +// t.Errorf("GetPrometheusImagesFromProfile() got = %v, want %v", got, tt.want) +// } +// }) +// } +//} +// +//func TestGetUnique(t *testing.T) { +// type args struct { +// all []*ImageInfo +// } +// tests := []struct { +// name string +// args args +// want []*ImageInfo +// }{ +// // TODO: Add test cases. +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// if got := GetUnique(tt.args.all); !reflect.DeepEqual(got, tt.want) { +// t.Errorf("GetUnique() = %v, want %v", got, tt.want) +// } +// }) +// } +//} diff --git a/cmd/root.go b/cmd/root.go index 18da302..324ae6e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,26 +13,25 @@ var ( dryRun bool verbose bool keepLast int - profileName string + arn string region string ) func init() { - + // rootCmd.PersistentFlags().StringVarP(&arn, "arn", "a", "arn:aws:iam::722014088219:role/devops-read-only", "The AWS profile to use (default: 'arn:aws:iam::722014088219:role/devops-read-only')") rootCmd.PersistentFlags().BoolVarP(&untaggedOnly, "untagged-only", "u", true, "Only delete untagged images (default: true)") rootCmd.PersistentFlags().IntVarP(&keepLast, "keep-latest-count", "k", 10, "Keep the latest X images (default: 10)") - rootCmd.PersistentFlags().StringVarP(&profileName, "profile-name", "p", "NA", "The AWS profile to use (default: '')") + rootCmd.PersistentFlags().StringVarP(&arn, "arn", "a", "arn:aws:iam::722014088219:role/devops-read-only", "The AWS profile to use (default: 'arn:aws:iam::722014088219:role/devops-read-only')") rootCmd.PersistentFlags().StringVarP(®ion, "region", "r", "us-east-1", "AWS region (default: us-east-1)") rootCmd.PersistentFlags().BoolVarP(&dryRun, "dry-run", "d", true, "Dry run of the clean action (default: true)") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Use to see details of images to be deleted") - } var rootCmd = &cobra.Command{ Use: "ecr-cleaner", Short: "clean amazon elastic container registries", Run: func(cmd *cobra.Command, args []string) { - clean.CleanRepos(untaggedOnly, keepLast, profileName, region, dryRun, verbose) + clean.CleanRepos(untaggedOnly, keepLast, arn, region, dryRun, verbose) }, } diff --git a/go.mod b/go.mod index 1f156a3..eca1abb 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/ltvco/ecr-cleaner go 1.21.4 require ( + github.com/aws/aws-sdk-go-v2 v1.23.5 github.com/aws/aws-sdk-go-v2/config v1.25.12 github.com/aws/aws-sdk-go-v2/service/ecr v1.24.3 github.com/rs/zerolog v1.31.0 @@ -26,11 +27,8 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/sys v0.12.0 // indirect + golang.org/x/sys v0.16.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) -require ( - github.com/aws/aws-sdk-go-v2 v1.23.5 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect -) +require github.com/jmespath/go-jmespath v0.4.0 // indirect diff --git a/go.sum b/go.sum index fd5a303..1efa3ff 100644 --- a/go.sum +++ b/go.sum @@ -59,8 +59,9 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=