From 12146bf356a3b26176c47e3a013a713fd14f346d Mon Sep 17 00:00:00 2001 From: Matthis Date: Sat, 18 Nov 2023 22:08:38 +0100 Subject: [PATCH] feat: rework cache package - add gcs cache - add cache purge command (#750) * feat: rework cache pkg Signed-off-by: Matthis Holleville * feat: Completion of cache pkg rework. Added cache purge command. Signed-off-by: Matthis Holleville * doc: add purgin command note Signed-off-by: Matthis Holleville * fix: disable cache if noCache is set Signed-off-by: Matthis Holleville * feat: improve GetCacheConfiguration lisibility & transform add method to addOrUpdate Signed-off-by: Matthis Holleville * feat: transform server mode to work with new cache configuration Signed-off-by: Matthis Holleville * fix: use 'switch' instead 'if' to evaluate Cache from grpc Signed-off-by: Matthis Holleville * feat: add mutually exclusive flags for command options Signed-off-by: Matthis Holleville * doc: update readme.md Signed-off-by: Matthis Holleville * feat: return err on bucket creation failed Signed-off-by: Matthis Holleville * feat: update dependencies Signed-off-by: Matthis Holleville --------- Signed-off-by: Matthis Holleville Signed-off-by: Matthis --- README.md | 16 ++++- cmd/cache/add.go | 22 ++++-- cmd/cache/list.go | 22 ++++-- cmd/cache/purge.go | 54 +++++++++++++++ go.mod | 18 ++++- go.sum | 41 ++++++++++-- pkg/analysis/analysis.go | 8 ++- pkg/cache/azuresa_based.go | 107 ++++++++++++++++------------- pkg/cache/cache.go | 133 ++++++++++++++++++++----------------- pkg/cache/file_based.go | 39 ++++++++++- pkg/cache/gcs_based.go | 133 +++++++++++++++++++++++++++++++++++++ pkg/cache/s3_based.go | 103 ++++++++++++++++------------ pkg/cache/types.go | 14 ++++ pkg/server/config.go | 23 +++++-- 14 files changed, 555 insertions(+), 178 deletions(-) create mode 100644 cmd/cache/purge.go create mode 100644 pkg/cache/gcs_based.go create mode 100644 pkg/cache/types.go diff --git a/README.md b/README.md index b6215da82f..bdcb7815d3 100644 --- a/README.md +++ b/README.md @@ -595,19 +595,29 @@ _Adding a remote cache_ * AWS S3 * _As a prerequisite `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are required as environmental variables._ - * Configuration, ``` k8sgpt cache add --region --bucket ``` + * Configuration, ``` k8sgpt cache add s3 --region --bucket ``` * K8sGPT will create the bucket if it does not exist * Azure Storage * We support a number of [techniques](https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication?tabs=bash#2-authenticate-with-azure) to authenticate against Azure - * Configuration, ``` k8sgpt cache add --storageacc --container ``` + * Configuration, ``` k8sgpt cache add azure --storageacc --container ``` * K8sGPT assumes that the storage account already exist and it will create the container if it does not exist - * It's **users'** responsibility have to grant specific permissions to their identity in order to be able to upload blob files and create SA containers (e.g Storage Blob Data Contributor) + * It is the **user** responsibility have to grant specific permissions to their identity in order to be able to upload blob files and create SA containers (e.g Storage Blob Data Contributor) + * Google Cloud Storage + * _As a prerequisite `GOOGLE_APPLICATION_CREDENTIALS` are required as environmental variables._ + * Configuration, ``` k8sgpt cache add gcs --region --bucket --projectid ``` + * K8sGPT will create the bucket if it does not exist _Listing cache items_ ``` k8sgpt cache list ``` +_Purging an object from the cache_ +Note: purging an object using this command will delete upstream files, so it requires appropriate permissions. +``` +k8sgpt cache purge $OBJECT_NAME +``` + _Removing the remote cache_ Note: this will not delete the upstream S3 bucket or Azure storage container ``` diff --git a/cmd/cache/add.go b/cmd/cache/add.go index 635ae5cb0a..979aa8f036 100644 --- a/cmd/cache/add.go +++ b/cmd/cache/add.go @@ -28,20 +28,31 @@ var ( bucketName string storageAccount string containerName string + projectId string ) // addCmd represents the add command var addCmd = &cobra.Command{ - Use: "add", + Use: "add [cache type]", Short: "Add a remote cache", Long: `This command allows you to add a remote cache to store the results of an analysis. The supported cache types are: - Azure Blob storage + - Google Cloud storage - S3`, Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + color.Red("Error: Please provide a value for cache types. Run k8sgpt cache add --help") + os.Exit(1) + } fmt.Println(color.YellowString("Adding remote based cache")) - remoteCache := cache.NewCacheProvider(bucketname, region, storageAccount, containerName) - err := cache.AddRemoteCache(remoteCache) + cacheType := args[0] + remoteCache, err := cache.NewCacheProvider(cacheType, bucketname, region, storageAccount, containerName, projectId) + if err != nil { + color.Red("Error: %v", err) + os.Exit(1) + } + err = cache.AddRemoteCache(remoteCache) if err != nil { color.Red("Error: %v", err) os.Exit(1) @@ -51,9 +62,10 @@ var addCmd = &cobra.Command{ func init() { CacheCmd.AddCommand(addCmd) - addCmd.Flags().StringVarP(®ion, "region", "r", "", "The region to use for the AWS S3 cache") + addCmd.Flags().StringVarP(®ion, "region", "r", "", "The region to use for the AWS S3 or GCS cache") addCmd.Flags().StringVarP(&bucketname, "bucket", "b", "", "The name of the AWS S3 bucket to use for the cache") addCmd.MarkFlagsRequiredTogether("region", "bucket") + addCmd.Flags().StringVarP(&projectId, "projectid", "p", "", "The GCP project ID") addCmd.Flags().StringVarP(&storageAccount, "storageacc", "s", "", "The Azure storage account name of the container") addCmd.Flags().StringVarP(&containerName, "container", "c", "", "The Azure container name to use for the cache") addCmd.MarkFlagsRequiredTogether("storageacc", "container") @@ -62,4 +74,6 @@ func init() { addCmd.MarkFlagsMutuallyExclusive("region", "container") addCmd.MarkFlagsMutuallyExclusive("bucket", "storageacc") addCmd.MarkFlagsMutuallyExclusive("bucket", "container") + addCmd.MarkFlagsMutuallyExclusive("projectid", "storageacc") + addCmd.MarkFlagsMutuallyExclusive("projectid", "container") } diff --git a/cmd/cache/list.go b/cmd/cache/list.go index 493515f543..5e3d320de2 100644 --- a/cmd/cache/list.go +++ b/cmd/cache/list.go @@ -16,9 +16,11 @@ package cache import ( "os" + "reflect" "github.com/fatih/color" "github.com/k8sgpt-ai/k8sgpt/pkg/cache" + "github.com/olekukonko/tablewriter" "github.com/spf13/cobra" ) @@ -30,22 +32,32 @@ var listCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { // load remote cache if it is configured - remoteCacheEnabled, err := cache.RemoteCacheEnabled() + c, err := cache.GetCacheConfiguration() if err != nil { color.Red("Error: %v", err) os.Exit(1) } - c := cache.New(false, remoteCacheEnabled) - // list the contents of the cache names, err := c.List() if err != nil { color.Red("Error: %v", err) os.Exit(1) } - for _, name := range names { - println(name) + var headers []string + obj := cache.CacheObjectDetails{} + objType := reflect.TypeOf(obj) + for i := 0; i < objType.NumField(); i++ { + field := objType.Field(i) + headers = append(headers, field.Name) } + + table := tablewriter.NewWriter(os.Stdout) + table.SetHeader(headers) + + for _, v := range names { + table.Append([]string{v.Name, v.UpdatedAt.String()}) + } + table.Render() }, } diff --git a/cmd/cache/purge.go b/cmd/cache/purge.go new file mode 100644 index 0000000000..63da9e0092 --- /dev/null +++ b/cmd/cache/purge.go @@ -0,0 +1,54 @@ +/* +Copyright 2023 The K8sGPT Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cache + +import ( + "fmt" + "os" + + "github.com/fatih/color" + "github.com/k8sgpt-ai/k8sgpt/pkg/cache" + "github.com/spf13/cobra" +) + +var purgeCmd = &cobra.Command{ + Use: "purge [object name]", + Short: "Purge a remote cache", + Long: "This command allows you to delete/purge one object from the cache", + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + color.Red("Error: Please provide a value for object name. Run k8sgpt cache purge --help") + os.Exit(1) + } + objectKey := args[0] + fmt.Println(color.YellowString("Purging a remote cache.")) + c, err := cache.GetCacheConfiguration() + if err != nil { + color.Red("Error: %v", err) + os.Exit(1) + } + + err = c.Remove(objectKey) + if err != nil { + color.Red("Error: %v", err) + os.Exit(1) + } + fmt.Println(color.GreenString("Object deleted.")) + }, +} + +func init() { + CacheCmd.AddCommand(purgeCmd) +} diff --git a/go.mod b/go.mod index 17526e616b..2adea16e04 100644 --- a/go.mod +++ b/go.mod @@ -24,15 +24,22 @@ require ( require github.com/adrg/xdg v0.4.0 require ( - buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1 - buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1 + buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231116211251-9f5041346631.2 + buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231116211251-9f5041346631.4 + cloud.google.com/go/storage v1.30.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 github.com/aws/aws-sdk-go v1.48.0 github.com/cohere-ai/cohere-go v0.2.0 + github.com/olekukonko/tablewriter v0.0.5 + google.golang.org/api v0.143.0 ) require ( + cloud.google.com/go v0.110.7 // indirect + cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect @@ -41,13 +48,20 @@ require ( github.com/cohere-ai/tokenizer v1.1.1 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect + go.opencensus.io v0.24.0 // indirect + google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect ) diff --git a/go.sum b/go.sum index a86c472392..1a06d00624 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,7 @@ -buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1 h1:xYEAhcdWh89HNtbM5Uv4b2xu+4/MkNffR9JNrnnEjXU= -buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1/go.mod h1:j2QJ3L7VTtI+VeK6t03h9X934FolVTb3FwXUc76bQMQ= -buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231002095256-194bc640518b.4/go.mod h1:i/s4ALHwKvjA1oGNKpoHg0FpEOTbufoOm/NdTE6YQAE= -buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1 h1:Bt8mnCodD/BqChxt/r3xYayGLoOAn334qC1tN7VqUTE= -buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1/go.mod h1:gtnk2yAUexdY5nTuUg0SH5WCCGvpKzr7pd3Xbi7MWjE= +buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231116211251-9f5041346631.2 h1:lr4kHFzPIzoWIHF6s/B+0Wb/WknraHrbqNguavOzP70= +buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231116211251-9f5041346631.2/go.mod h1:g/UdFu0wAAS44ncCfX3zjna7WC1gp6BENVL6LPJ9tow= +buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231116211251-9f5041346631.4 h1:f7e94G/tYVs+QDTvjU9sFolC5tuJxlOjzUBJ84XMwkg= +buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231116211251-9f5041346631.4/go.mod h1:i/s4ALHwKvjA1oGNKpoHg0FpEOTbufoOm/NdTE6YQAE= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -41,6 +40,8 @@ cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRY cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= +cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= @@ -176,9 +177,12 @@ cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvj cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= @@ -313,6 +317,8 @@ cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGE cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= @@ -531,6 +537,8 @@ cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeL cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= @@ -839,6 +847,7 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -901,10 +910,13 @@ github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/licenseclassifier/v2 v2.0.0 h1:1Y57HHILNf4m0ABuMVb6xk4vAJYEUO0gDxNpog0pyeA= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -923,6 +935,8 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3 h1:mpL/HvfIgIejhVwAfxBQkwEjlhP5o0O9RAeTAjpwzxc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -935,6 +949,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -946,6 +962,8 @@ github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqE github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= @@ -1052,6 +1070,7 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -1098,8 +1117,10 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/gomega v1.28.1 h1:MijcGUbfYuznzK/5R4CPNoUP/9Xvuo20sXfEm6XxoTA= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= @@ -1705,6 +1726,8 @@ google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/ google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.143.0 h1:o8cekTkqhywkbZT6p1UHJPZ9+9uuCAJs/KYomxZB8fA= +google.golang.org/api v0.143.0/go.mod h1:FoX9DO9hT7DLNn97OuoZAGSDuNAXdJRuGK98rSUgurk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1846,8 +1869,12 @@ google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOl google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb h1:XFBgcDwm7irdHTbz4Zk2h7Mh+eis4nfJEFQFYzJzuIA= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM= diff --git a/pkg/analysis/analysis.go b/pkg/analysis/analysis.go index aab590f96c..eea5476c95 100644 --- a/pkg/analysis/analysis.go +++ b/pkg/analysis/analysis.go @@ -115,18 +115,22 @@ func NewAnalysis(backend string, language string, filters []string, namespace st } // load remote cache if it is configured - remoteCacheEnabled, err := cache.RemoteCacheEnabled() + cache, err := cache.GetCacheConfiguration() if err != nil { return nil, err } + if noCache { + cache.DisableCache() + } + return &Analysis{ Context: ctx, Filters: filters, Client: client, AIClient: aiClient, Namespace: namespace, - Cache: cache.New(noCache, remoteCacheEnabled), + Cache: cache, Explain: explain, MaxConcurrency: maxConcurrency, AnalysisAIProvider: backend, diff --git a/pkg/cache/azuresa_based.go b/pkg/cache/azuresa_based.go index c2bda42ae6..89e1fa5cd3 100644 --- a/pkg/cache/azuresa_based.go +++ b/pkg/cache/azuresa_based.go @@ -9,7 +9,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/spf13/viper" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" ) // Generate ICache implementation @@ -20,6 +20,48 @@ type AzureCache struct { session *azblob.Client } +type AzureCacheConfiguration struct { + StorageAccount string `mapstructure:"storageaccount" yaml:"storageaccount,omitempty"` + ContainerName string `mapstructure:"container" yaml:"container,omitempty"` +} + +func (s *AzureCache) Configure(cacheInfo CacheProvider) error { + s.ctx = context.Background() + if cacheInfo.Azure.ContainerName == "" { + log.Fatal("Azure Container name not configured") + } + if cacheInfo.Azure.StorageAccount == "" { + log.Fatal("Azure Storage account not configured") + } + + // We assume that Storage account is already in place + blobUrl := fmt.Sprintf("https://%s.blob.core.windows.net/", cacheInfo.Azure.StorageAccount) + credential, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + log.Fatal(err) + } + client, err := azblob.NewClient(blobUrl, credential, nil) + if err != nil { + log.Fatal(err) + } + // Try to create the blob container + _, err = client.CreateContainer(s.ctx, cacheInfo.Azure.ContainerName, nil) + if err != nil { + // TODO: Maybe there is a better way to check this? + // docs: https://pkg.go.dev/github.com/Azure/azure-storage-blob-go/azblob + if strings.Contains(err.Error(), "ContainerAlreadyExists") { + // do nothing + } else { + return err + } + } + s.containerName = cacheInfo.Azure.ContainerName + s.session = client + + return nil + +} + func (s *AzureCache) Store(key string, data string) error { // Store the object as a new file in the Azure blob storage with data as the content cacheData := []byte(data) @@ -45,9 +87,9 @@ func (s *AzureCache) Load(key string) (string, error) { return data.String(), nil } -func (s *AzureCache) List() ([]string, error) { +func (s *AzureCache) List() ([]CacheObjectDetails, error) { // List the files in the blob containerName - files := []string{} + files := []CacheObjectDetails{} pager := s.session.NewListBlobsFlatPager(s.containerName, &azblob.ListBlobsFlatOptions{ Include: azblob.ListBlobsInclude{Snapshots: false, Versions: false}, @@ -60,13 +102,24 @@ func (s *AzureCache) List() ([]string, error) { } for _, blob := range resp.Segment.BlobItems { - files = append(files, *blob.Name) + files = append(files, CacheObjectDetails{ + Name: *blob.Name, + UpdatedAt: *blob.Properties.LastModified, + }) } } return files, nil } +func (s *AzureCache) Remove(key string) error { + _, err := s.session.DeleteBlob(s.ctx, s.containerName, key, &blob.DeleteOptions{}) + if err != nil { + return err + } + return nil +} + func (s *AzureCache) Exists(key string) bool { // Check if the object exists in the blob storage pager := s.session.NewListBlobsFlatPager(s.containerName, &azblob.ListBlobsFlatOptions{ @@ -93,46 +146,10 @@ func (s *AzureCache) IsCacheDisabled() bool { return s.noCache } -func NewAzureCache(nocache bool) ICache { - ctx := context.Background() - var cache CacheProvider - err := viper.UnmarshalKey("cache", &cache) - if err != nil { - panic(err) - } - if cache.ContainerName == "" { - log.Fatal("Azure Container name not configured") - } - if cache.StorageAccount == "" { - log.Fatal("Azure Storage account not configured") - } - - // We assume that Storage account is already in place - blobUrl := fmt.Sprintf("https://%s.blob.core.windows.net/", cache.StorageAccount) - credential, err := azidentity.NewDefaultAzureCredential(nil) - if err != nil { - log.Fatal(err) - } - client, err := azblob.NewClient(blobUrl, credential, nil) - if err != nil { - log.Fatal(err) - } - // Try to create the blob container - _, err = client.CreateContainer(ctx, cache.ContainerName, nil) - if err != nil { - // TODO: Maybe there is a better way to check this? - // docs: https://pkg.go.dev/github.com/Azure/azure-storage-blob-go/azblob - if strings.Contains(err.Error(), "ContainerAlreadyExists") { - // do nothing - } else { - log.Fatal(err) - } - } +func (s *AzureCache) GetName() string { + return "azure" +} - return &AzureCache{ - ctx: ctx, - noCache: nocache, - containerName: cache.ContainerName, - session: client, - } +func (s *AzureCache) DisableCache() { + s.noCache = true } diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index f4d96d381e..dcb0b0b117 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -1,92 +1,108 @@ package cache import ( - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" + "fmt" "github.com/spf13/viper" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) -type CacheType string - -const ( - Azure CacheType = "azure" - S3 CacheType = "s3" - FileBased CacheType = "file" +var ( + types = []ICache{ + &AzureCache{}, + &FileBasedCache{}, + &GCSCache{}, + &S3Cache{}, + } ) type ICache interface { + Configure(cacheInfo CacheProvider) error Store(key string, data string) error Load(key string) (string, error) - List() ([]string, error) + List() ([]CacheObjectDetails, error) + Remove(key string) error Exists(key string) bool IsCacheDisabled() bool + GetName() string + DisableCache() } -func New(noCache bool, remoteCache CacheType) ICache { - switch remoteCache { - case S3: - return NewS3Cache(noCache) - case Azure: - return NewAzureCache(noCache) - case FileBased: - return &FileBasedCache{ - noCache: noCache, - } - default: - return &FileBasedCache{ - noCache: noCache, +func New(cacheType string) ICache { + for _, t := range types { + if cacheType == t.GetName() { + return t } } + return &FileBasedCache{} } -// CacheProvider is the configuration for the cache provider when using a remote cache -type CacheProvider struct { - BucketName string `mapstructure:"bucketname" yaml:"bucketname,omitempty"` - Region string `mapstructure:"region" yaml:"region,omitempty"` - StorageAccount string `mapstructure:"storageaccount" yaml:"storageaccount,omitempty"` - ContainerName string `mapstructure:"container" yaml:"container,omitempty"` +func ParseCacheConfiguration() (CacheProvider, error) { + var cacheInfo CacheProvider + err := viper.UnmarshalKey("cache", &cacheInfo) + if err != nil { + return cacheInfo, err + } + return cacheInfo, nil } -// NewCacheProvider constructs a new cache struct -func NewCacheProvider(bucketname, region, storageaccount, containername string) CacheProvider { - return CacheProvider{ - BucketName: bucketname, - Region: region, - StorageAccount: storageaccount, - ContainerName: containername, +func NewCacheProvider(cacheType, bucketname, region, storageAccount, containerName, projectId string) (CacheProvider, error) { + cProvider := CacheProvider{} + + switch { + case cacheType == "azure": + cProvider.Azure.ContainerName = containerName + cProvider.Azure.StorageAccount = storageAccount + case cacheType == "gcs": + cProvider.GCS.BucketName = bucketname + cProvider.GCS.ProjectId = projectId + cProvider.GCS.Region = region + case cacheType == "s3": + cProvider.S3.BucketName = bucketname + cProvider.S3.Region = region + default: + return CacheProvider{}, status.Error(codes.Internal, fmt.Sprintf("%s is not a valid option", cacheType)) } -} -// If we have set a remote cache, return the remote cache type -func RemoteCacheEnabled() (CacheType, error) { - // load remote cache if it is configured - var cache CacheProvider - err := viper.UnmarshalKey("cache", &cache) + cache := New(cacheType) + err := cache.Configure(cProvider) if err != nil { - return "", err + return CacheProvider{}, err } - if cache.BucketName != "" && cache.Region != "" { - return S3, nil - } else if cache.StorageAccount != "" && cache.ContainerName != "" { - return Azure, nil - } - return FileBased, nil + return cProvider, nil } -func AddRemoteCache(cache CacheProvider) error { - var cacheInfo CacheProvider - err := viper.UnmarshalKey("cache", &cacheInfo) +// If we have set a remote cache, return the remote cache configuration +func GetCacheConfiguration() (ICache, error) { + cacheInfo, err := ParseCacheConfiguration() if err != nil { - return err + return nil, err + } + + var cache ICache + + switch { + case cacheInfo.GCS != GCSCacheConfiguration{}: + cache = &GCSCache{} + case cacheInfo.Azure != AzureCacheConfiguration{}: + cache = &AzureCache{} + case cacheInfo.S3 != S3CacheConfiguration{}: + cache = &S3Cache{} + default: + cache = &FileBasedCache{} } - cacheInfo.BucketName = cache.BucketName - cacheInfo.Region = cache.Region - cacheInfo.StorageAccount = cache.StorageAccount - cacheInfo.ContainerName = cache.ContainerName + cache.Configure(cacheInfo) + + return cache, nil +} + +func AddRemoteCache(cacheInfo CacheProvider) error { + viper.Set("cache", cacheInfo) - err = viper.WriteConfig() + + err := viper.WriteConfig() if err != nil { return err } @@ -99,9 +115,6 @@ func RemoveRemoteCache() error { if err != nil { return status.Error(codes.Internal, "cache unmarshal") } - if cacheInfo.BucketName == "" && cacheInfo.ContainerName == "" && cacheInfo.StorageAccount == "" { - return status.Error(codes.Internal, "no remote cache configured") - } cacheInfo = CacheProvider{} viper.Set("cache", cacheInfo) diff --git a/pkg/cache/file_based.go b/pkg/cache/file_based.go index 9e59886bda..66926a7e85 100644 --- a/pkg/cache/file_based.go +++ b/pkg/cache/file_based.go @@ -15,11 +15,15 @@ type FileBasedCache struct { noCache bool } +func (f *FileBasedCache) Configure(cacheInfo CacheProvider) error { + return nil +} + func (f *FileBasedCache) IsCacheDisabled() bool { return f.noCache } -func (*FileBasedCache) List() ([]string, error) { +func (*FileBasedCache) List() ([]CacheObjectDetails, error) { path, err := xdg.CacheFile("k8sgpt") if err != nil { return nil, err @@ -30,9 +34,16 @@ func (*FileBasedCache) List() ([]string, error) { return nil, err } - var result []string + var result []CacheObjectDetails for _, file := range files { - result = append(result, file.Name()) + info, err := file.Info() + if err != nil { + return nil, err + } + result = append(result, CacheObjectDetails{ + Name: file.Name(), + UpdatedAt: info.ModTime(), + }) } return result, nil @@ -72,6 +83,20 @@ func (*FileBasedCache) Load(key string) (string, error) { return string(data), nil } +func (*FileBasedCache) Remove(key string) error { + path, err := xdg.CacheFile(filepath.Join("k8sgpt", key)) + + if err != nil { + return err + } + + if err := os.Remove(path); err != nil { + return err + } + + return nil +} + func (*FileBasedCache) Store(key string, data string) error { path, err := xdg.CacheFile(filepath.Join("k8sgpt", key)) @@ -81,3 +106,11 @@ func (*FileBasedCache) Store(key string, data string) error { return os.WriteFile(path, []byte(data), 0600) } + +func (s *FileBasedCache) GetName() string { + return "file" +} + +func (s *FileBasedCache) DisableCache() { + s.noCache = true +} diff --git a/pkg/cache/gcs_based.go b/pkg/cache/gcs_based.go new file mode 100644 index 0000000000..75689f8e2a --- /dev/null +++ b/pkg/cache/gcs_based.go @@ -0,0 +1,133 @@ +package cache + +import ( + "context" + "io" + "log" + + "cloud.google.com/go/storage" + "google.golang.org/api/iterator" +) + +type GCSCache struct { + ctx context.Context + noCache bool + bucketName string + projectId string + region string + session *storage.Client +} + +type GCSCacheConfiguration struct { + ProjectId string `mapstructure:"projectid" yaml:"projectid,omitempty"` + Region string `mapstructure:"region" yaml:"region,omitempty"` + BucketName string `mapstructure:"bucketname" yaml:"bucketname,omitempty"` +} + +func (s *GCSCache) Configure(cacheInfo CacheProvider) error { + s.ctx = context.Background() + if cacheInfo.GCS.BucketName == "" { + log.Fatal("Bucket name not configured") + } + if cacheInfo.GCS.Region == "" { + log.Fatal("Region not configured") + } + if cacheInfo.GCS.ProjectId == "" { + log.Fatal("ProjectID not configured") + } + s.bucketName = cacheInfo.GCS.BucketName + s.projectId = cacheInfo.GCS.ProjectId + s.region = cacheInfo.GCS.Region + storageClient, err := storage.NewClient(s.ctx) + if err != nil { + log.Fatal(err) + } + + _, err = storageClient.Bucket(s.bucketName).Attrs(s.ctx) + if err == storage.ErrBucketNotExist { + err = storageClient.Bucket(s.bucketName).Create(s.ctx, s.projectId, &storage.BucketAttrs{ + Location: s.region, + }) + if err != nil { + return err + } + } + s.session = storageClient + return nil +} + +func (s *GCSCache) Store(key string, data string) error { + wc := s.session.Bucket(s.bucketName).Object(key).NewWriter(s.ctx) + + if _, err := wc.Write([]byte(data)); err != nil { + return err + } + + if err := wc.Close(); err != nil { + return err + } + + return nil +} + +func (s *GCSCache) Load(key string) (string, error) { + reader, err := s.session.Bucket(s.bucketName).Object(key).NewReader(s.ctx) + if err != nil { + return "", err + } + defer reader.Close() + + data, err := io.ReadAll(reader) + if err != nil { + return "", err + } + + return string(data), nil +} + +func (s *GCSCache) Remove(key string) error { + bucketClient := s.session.Bucket(s.bucketName) + obj := bucketClient.Object(key) + if err := obj.Delete(s.ctx); err != nil { + return err + } + return nil +} + +func (s *GCSCache) List() ([]CacheObjectDetails, error) { + var files []CacheObjectDetails + + items := s.session.Bucket(s.bucketName).Objects(s.ctx, nil) + for { + attrs, err := items.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, err + } + files = append(files, CacheObjectDetails{ + Name: attrs.Name, + UpdatedAt: attrs.Updated, + }) + } + return files, nil +} + +func (s *GCSCache) Exists(key string) bool { + obj := s.session.Bucket(s.bucketName).Object(key) + _, err := obj.Attrs(s.ctx) + return err == nil +} + +func (s *GCSCache) IsCacheDisabled() bool { + return s.noCache +} + +func (s *GCSCache) GetName() string { + return "gcs" +} + +func (s *GCSCache) DisableCache() { + s.noCache = true +} diff --git a/pkg/cache/s3_based.go b/pkg/cache/s3_based.go index 444946ef83..19e4f46d49 100644 --- a/pkg/cache/s3_based.go +++ b/pkg/cache/s3_based.go @@ -7,7 +7,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" - "github.com/spf13/viper" ) // Generate ICache implementation @@ -17,6 +16,45 @@ type S3Cache struct { session *s3.S3 } +type S3CacheConfiguration struct { + Region string `mapstructure:"region" yaml:"region,omitempty"` + BucketName string `mapstructure:"bucketname" yaml:"bucketname,omitempty"` +} + +func (s *S3Cache) Configure(cacheInfo CacheProvider) error { + if cacheInfo.S3.BucketName == "" { + log.Fatal("Bucket name not configured") + } + if cacheInfo.S3.Region == "" { + log.Fatal("Region not configured") + } + s.bucketName = cacheInfo.S3.BucketName + + sess := session.Must(session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + Config: aws.Config{ + Region: aws.String(cacheInfo.S3.Region), + }, + })) + + s3Client := s3.New(sess) + + // Check if the bucket exists, if not create it + _, err := s3Client.HeadBucket(&s3.HeadBucketInput{ + Bucket: aws.String(cacheInfo.S3.BucketName), + }) + if err != nil { + _, err = s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(cacheInfo.S3.BucketName), + }) + if err != nil { + return err + } + } + s.session = s3Client + return nil +} + func (s *S3Cache) Store(key string, data string) error { // Store the object as a new file in the bucket with data as the content _, err := s.session.PutObject(&s3.PutObjectInput{ @@ -28,6 +66,18 @@ func (s *S3Cache) Store(key string, data string) error { } +func (s *S3Cache) Remove(key string) error { + _, err := s.session.DeleteObject(&s3.DeleteObjectInput{ + Bucket: &s.bucketName, + Key: aws.String(key), + }) + + if err != nil { + return err + } + return nil +} + func (s *S3Cache) Load(key string) (string, error) { // Retrieve the object from the bucket and load it into a string @@ -45,7 +95,7 @@ func (s *S3Cache) Load(key string) (string, error) { return buf.String(), nil } -func (s *S3Cache) List() ([]string, error) { +func (s *S3Cache) List() ([]CacheObjectDetails, error) { // List the files in the bucket result, err := s.session.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(s.bucketName)}) @@ -53,9 +103,12 @@ func (s *S3Cache) List() ([]string, error) { return nil, err } - var keys []string + var keys []CacheObjectDetails for _, item := range result.Contents { - keys = append(keys, *item.Key) + keys = append(keys, CacheObjectDetails{ + Name: *item.Key, + UpdatedAt: *item.LastModified, + }) } return keys, nil @@ -75,42 +128,10 @@ func (s *S3Cache) IsCacheDisabled() bool { return s.noCache } -func NewS3Cache(nocache bool) ICache { - - var cache CacheProvider - err := viper.UnmarshalKey("cache", &cache) - if err != nil { - log.Fatal(err) - } - if cache.BucketName == "" { - log.Fatal("Bucket name not configured") - } - if cache.Region == "" { - log.Fatal("Region not configured") - } - - sess := session.Must(session.NewSessionWithOptions(session.Options{ - SharedConfigState: session.SharedConfigEnable, - Config: aws.Config{ - Region: aws.String(cache.Region), - }, - })) - - s := s3.New(sess) - - // Check if the bucket exists, if not create it - _, err = s.HeadBucket(&s3.HeadBucketInput{ - Bucket: aws.String(cache.BucketName), - }) - if err != nil { - _, _ = s.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(cache.BucketName), - }) - } +func (s *S3Cache) GetName() string { + return "s3" +} - return &S3Cache{ - noCache: nocache, - session: s, - bucketName: cache.BucketName, - } +func (s *S3Cache) DisableCache() { + s.noCache = true } diff --git a/pkg/cache/types.go b/pkg/cache/types.go new file mode 100644 index 0000000000..1e4bee976a --- /dev/null +++ b/pkg/cache/types.go @@ -0,0 +1,14 @@ +package cache + +import "time" + +type CacheProvider struct { + GCS GCSCacheConfiguration `mapstructucre:"gcs" yaml:"gcs,omitempty"` + Azure AzureCacheConfiguration `mapstructucre:"azure" yaml:"azure,omitempty"` + S3 S3CacheConfiguration `mapstructucre:"s3" yaml:"s3,omitempty"` +} + +type CacheObjectDetails struct { + Name string + UpdatedAt time.Time +} diff --git a/pkg/server/config.go b/pkg/server/config.go index 26525c047e..3425878d5e 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -18,17 +18,28 @@ func (h *handler) AddConfig(ctx context.Context, i *schemav1.AddConfigRequest) ( } if i.Cache != nil { - // We check if we have a mixed cache configuration - CacheConfigured := (i.Cache.Region == "" && i.Cache.BucketName == "") || (i.Cache.ContainerName == "" && i.Cache.StorageAccount == "") - if !CacheConfigured { - return resp, status.Error(codes.InvalidArgument, "mixed cache arguments") + var err error + var remoteCache cache.CacheProvider + + switch i.Cache.GetCacheType().(type) { + case *schemav1.Cache_AzureCache: + remoteCache, err = cache.NewCacheProvider("azure", "", "", i.Cache.GetAzureCache().StorageAccount, i.Cache.GetAzureCache().ContainerName, "") + case *schemav1.Cache_S3Cache: + remoteCache, err = cache.NewCacheProvider("s3", i.Cache.GetS3Cache().BucketName, i.Cache.GetS3Cache().Region, "", "", "") + case *schemav1.Cache_GcsCache: + remoteCache, err = cache.NewCacheProvider("gcs", i.Cache.GetGcsCache().BucketName, i.Cache.GetGcsCache().Region, "", "", i.Cache.GetGcsCache().GetProjectId()) + default: + return resp, status.Error(codes.InvalidArgument, "Invalid cache configuration") } - cacheProvider := cache.NewCacheProvider(i.Cache.BucketName, i.Cache.Region, i.Cache.StorageAccount, i.Cache.ContainerName) - err := cache.AddRemoteCache(cacheProvider) if err != nil { return resp, err } + err = cache.AddRemoteCache(remoteCache) + if err != nil { + return resp, err + } + } return resp, nil }