From 23ac52d5ffc0b2ebb7516b070fa740108cb4299a Mon Sep 17 00:00:00 2001 From: Aris Boutselis Date: Sun, 22 Oct 2023 18:08:39 +0300 Subject: [PATCH] feat: add Azure remote cache (#690) * feat: add Azure remote cache Signed-off-by: Aris Boutselis * feat: add serve mode support and update buf schema Signed-off-by: Aris Boutselis * fix: map structure name Signed-off-by: Aris Boutselis * chore: add a new cache type to make code readable Signed-off-by: Aris Boutselis * docs: update docs to reflect new remote cache type Signed-off-by: Aris Boutselis * fix(deps): update module github.com/prometheus/client_golang to v1.17.0 (#687) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis * fix(deps): update module buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go to v1.3.0-20231002095256-194bc640518b.1 (#692) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix(deps): update module helm.sh/helm/v3 to v3.13.0 (#688) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix: security warning around printing provider details in https://github.com/k8sgpt-ai/k8sgpt/security/code-scanning/1 (#695) Signed-off-by: Aris Boutselis * fix(deps): update module buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go to v1.31.0-20231002095256-194bc640518b.1 (#693) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix(deps): update module github.com/sashabaranov/go-openai to v1.15.4 (#689) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis * fix(deps): update module github.com/aws/aws-sdk-go to v1.45.20 (#685) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * chore(deps): update amannn/action-semantic-pull-request action to v5.3.0 (#683) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis * fix(deps): update module github.com/aws/aws-sdk-go to v1.45.21 (#696) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * fix(deps): update module github.com/aws/aws-sdk-go to v1.45.22 (#697) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis * fix(deps): update module github.com/aws/aws-sdk-go to v1.45.23 (#699) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis * fix(deps): update module github.com/aws/aws-sdk-go to v1.45.24 (#701) Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis --------- Signed-off-by: Aris Boutselis Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Signed-off-by: Aris Boutselis Co-authored-by: Aris Boutselis Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Alex Jones --- README.md | 24 ++++--- cmd/cache/add.go | 27 +++++--- cmd/cache/remove.go | 2 +- go.mod | 8 +++ go.sum | 25 +++++++ pkg/cache/azuresa_based.go | 138 +++++++++++++++++++++++++++++++++++++ pkg/cache/cache.go | 67 +++++++++++++----- pkg/cache/s3_based.go | 7 +- pkg/server/config.go | 15 ++-- 9 files changed, 268 insertions(+), 45 deletions(-) create mode 100644 pkg/cache/azuresa_based.go diff --git a/README.md b/README.md index 3fde2c4eac..5b676f478b 100644 --- a/README.md +++ b/README.md @@ -485,18 +485,22 @@ Config file locations:
There may be scenarios where caching remotely is preferred. -In these scenarios K8sGPT supports AWS S3 Integration. +In these scenarios K8sGPT supports AWS S3 or Azure Blob storage Integration. - Remote caching - - _As a prerequisite `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are required as environmental variables._ + Remote caching +Note: You can only configure and use only one remote cache at a time _Adding a remote cache_ -Note: this will create the bucket if it does not exist -``` -k8sgpt cache add --region --bucket -``` + * AWS S3 + * _As a prerequisite `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` are required as environmental variables._ + * Configuration, ``` k8sgpt cache add --region --bucket ``` + * K8sGPT will create the bucket if it does not exist + * Azure Storage + * We support a number of [techniques](https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication?tabs=bash#2-authenticate-with-azure) to authenticate against Azure + * Configuration, ``` k8sgpt cache add --storageacc --container ``` + * K8sGPT assumes that the storage account already exist and it will create the container if it does not exist + * It's **users'** responsibility have to grant specific permissions to their identity in order to be able to upload blob files and create SA containers (e.g Storage Blob Data Contributor) _Listing cache items_ ``` @@ -504,9 +508,9 @@ k8sgpt cache list ``` _Removing the remote cache_ -Note: this will not delete the bucket +Note: this will not delete the upstream S3 bucket or Azure storage container ``` -k8sgpt cache remove --bucket +k8sgpt cache remove ```
diff --git a/cmd/cache/add.go b/cmd/cache/add.go index b97f5f2197..635ae5cb0a 100644 --- a/cmd/cache/add.go +++ b/cmd/cache/add.go @@ -24,7 +24,10 @@ import ( ) var ( - region string + region string + bucketName string + storageAccount string + containerName string ) // addCmd represents the add command @@ -33,10 +36,12 @@ var addCmd = &cobra.Command{ Short: "Add a remote cache", Long: `This command allows you to add a remote cache to store the results of an analysis. The supported cache types are: + - Azure Blob storage - S3`, Run: func(cmd *cobra.Command, args []string) { - fmt.Println(color.YellowString("Adding remote S3 based cache")) - err := cache.AddRemoteCache(bucketname, region) + fmt.Println(color.YellowString("Adding remote based cache")) + remoteCache := cache.NewCacheProvider(bucketname, region, storageAccount, containerName) + err := cache.AddRemoteCache(remoteCache) if err != nil { color.Red("Error: %v", err) os.Exit(1) @@ -46,9 +51,15 @@ var addCmd = &cobra.Command{ func init() { CacheCmd.AddCommand(addCmd) - addCmd.Flags().StringVarP(®ion, "region", "r", "", "The region to use for the cache") - addCmd.Flags().StringVarP(&bucketname, "bucket", "b", "", "The name of the bucket to use for the cache") - addCmd.MarkFlagRequired("bucket") - addCmd.MarkFlagRequired("region") - + addCmd.Flags().StringVarP(®ion, "region", "r", "", "The region to use for the AWS S3 cache") + addCmd.Flags().StringVarP(&bucketname, "bucket", "b", "", "The name of the AWS S3 bucket to use for the cache") + addCmd.MarkFlagsRequiredTogether("region", "bucket") + addCmd.Flags().StringVarP(&storageAccount, "storageacc", "s", "", "The Azure storage account name of the container") + addCmd.Flags().StringVarP(&containerName, "container", "c", "", "The Azure container name to use for the cache") + addCmd.MarkFlagsRequiredTogether("storageacc", "container") + // Tedious check to ensure we don't include arguments from different providers + addCmd.MarkFlagsMutuallyExclusive("region", "storageacc") + addCmd.MarkFlagsMutuallyExclusive("region", "container") + addCmd.MarkFlagsMutuallyExclusive("bucket", "storageacc") + addCmd.MarkFlagsMutuallyExclusive("bucket", "container") } diff --git a/cmd/cache/remove.go b/cmd/cache/remove.go index 56fd5b73cd..0ab14171fd 100644 --- a/cmd/cache/remove.go +++ b/cmd/cache/remove.go @@ -29,7 +29,7 @@ var removeCmd = &cobra.Command{ Long: `This command allows you to remove the remote cache and use the default filecache.`, Run: func(cmd *cobra.Command, args []string) { - err := cache.RemoveRemoteCache(bucketname) + err := cache.RemoveRemoteCache() if err != nil { color.Red("Error: %v", err) os.Exit(1) diff --git a/go.mod b/go.mod index 7143759075..cf7db38f3b 100644 --- a/go.mod +++ b/go.mod @@ -26,17 +26,25 @@ require github.com/adrg/xdg v0.4.0 require ( buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1 buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/aws/aws-sdk-go v1.45.26 github.com/cohere-ai/cohere-go v0.2.0 ) require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/Microsoft/hcsshim v0.11.0 // indirect github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect github.com/cohere-ai/tokenizer v1.1.1 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect diff --git a/go.sum b/go.sum index 5bc863c3c6..529482cadb 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1 h1:xYEAhcdWh89HNtbM5Uv4b2xu+4/MkNffR9JNrnnEjXU= buf.build/gen/go/k8sgpt-ai/k8sgpt/grpc/go v1.3.0-20231002095256-194bc640518b.1/go.mod h1:j2QJ3L7VTtI+VeK6t03h9X934FolVTb3FwXUc76bQMQ= buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.28.1-20231002095256-194bc640518b.4/go.mod h1:i/s4ALHwKvjA1oGNKpoHg0FpEOTbufoOm/NdTE6YQAE= +buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20230927080702-a2be8a73637d.1 h1:Snnz9mUZNxMFpd+l5m1zaVdIVAplVmdFxYVn5/f4UoI= +buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20230927080702-a2be8a73637d.1/go.mod h1:gtnk2yAUexdY5nTuUg0SH5WCCGvpKzr7pd3Xbi7MWjE= buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1 h1:Bt8mnCodD/BqChxt/r3xYayGLoOAn334qC1tN7VqUTE= buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go v1.31.0-20231002095256-194bc640518b.1/go.mod h1:gtnk2yAUexdY5nTuUg0SH5WCCGvpKzr7pd3Xbi7MWjE= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= @@ -601,8 +603,22 @@ gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zum git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -722,6 +738,7 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE= github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= @@ -818,6 +835,9 @@ github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -1007,6 +1027,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= @@ -1103,6 +1125,8 @@ github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1486,6 +1510,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/cache/azuresa_based.go b/pkg/cache/azuresa_based.go new file mode 100644 index 0000000000..c2bda42ae6 --- /dev/null +++ b/pkg/cache/azuresa_based.go @@ -0,0 +1,138 @@ +package cache + +import ( + "bytes" + "context" + "fmt" + "log" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/spf13/viper" +) + +// Generate ICache implementation +type AzureCache struct { + ctx context.Context + noCache bool + containerName string + session *azblob.Client +} + +func (s *AzureCache) Store(key string, data string) error { + // Store the object as a new file in the Azure blob storage with data as the content + cacheData := []byte(data) + _, err := s.session.UploadBuffer(s.ctx, s.containerName, key, cacheData, &azblob.UploadBufferOptions{}) + return err +} + +func (s *AzureCache) Load(key string) (string, error) { + // Load blob file contents + load, err := s.session.DownloadStream(s.ctx, s.containerName, key, nil) + if err != nil { + return "", err + } + data := bytes.Buffer{} + retryReader := load.NewRetryReader(s.ctx, &azblob.RetryReaderOptions{}) + _, err = data.ReadFrom(retryReader) + if err != nil { + return "", err + } + if err := retryReader.Close(); err != nil { + return "", err + } + return data.String(), nil +} + +func (s *AzureCache) List() ([]string, error) { + // List the files in the blob containerName + files := []string{} + + pager := s.session.NewListBlobsFlatPager(s.containerName, &azblob.ListBlobsFlatOptions{ + Include: azblob.ListBlobsInclude{Snapshots: false, Versions: false}, + }) + + for pager.More() { + resp, err := pager.NextPage(s.ctx) + if err != nil { + return nil, err + } + + for _, blob := range resp.Segment.BlobItems { + files = append(files, *blob.Name) + } + } + + return files, nil +} + +func (s *AzureCache) Exists(key string) bool { + // Check if the object exists in the blob storage + pager := s.session.NewListBlobsFlatPager(s.containerName, &azblob.ListBlobsFlatOptions{ + Include: azblob.ListBlobsInclude{Snapshots: false, Versions: false}, + }) + + for pager.More() { + resp, err := pager.NextPage(s.ctx) + if err != nil { + return false + } + + for _, blob := range resp.Segment.BlobItems { + if *blob.Name == key { + return true + } + } + } + + return false +} + +func (s *AzureCache) IsCacheDisabled() bool { + return s.noCache +} + +func NewAzureCache(nocache bool) ICache { + ctx := context.Background() + var cache CacheProvider + err := viper.UnmarshalKey("cache", &cache) + if err != nil { + panic(err) + } + if cache.ContainerName == "" { + log.Fatal("Azure Container name not configured") + } + if cache.StorageAccount == "" { + log.Fatal("Azure Storage account not configured") + } + + // We assume that Storage account is already in place + blobUrl := fmt.Sprintf("https://%s.blob.core.windows.net/", cache.StorageAccount) + credential, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + log.Fatal(err) + } + client, err := azblob.NewClient(blobUrl, credential, nil) + if err != nil { + log.Fatal(err) + } + // Try to create the blob container + _, err = client.CreateContainer(ctx, cache.ContainerName, nil) + if err != nil { + // TODO: Maybe there is a better way to check this? + // docs: https://pkg.go.dev/github.com/Azure/azure-storage-blob-go/azblob + if strings.Contains(err.Error(), "ContainerAlreadyExists") { + // do nothing + } else { + log.Fatal(err) + } + } + + return &AzureCache{ + ctx: ctx, + noCache: nocache, + containerName: cache.ContainerName, + session: client, + } +} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 16858e980d..f4d96d381e 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -7,6 +7,14 @@ import ( "github.com/spf13/viper" ) +type CacheType string + +const ( + Azure CacheType = "azure" + S3 CacheType = "s3" + FileBased CacheType = "file" +) + type ICache interface { Store(key string, data string) error Load(key string) (string, error) @@ -15,43 +23,68 @@ type ICache interface { IsCacheDisabled() bool } -func New(noCache bool, remoteCache bool) ICache { - if remoteCache { +func New(noCache bool, remoteCache CacheType) ICache { + switch remoteCache { + case S3: return NewS3Cache(noCache) - } - return &FileBasedCache{ - noCache: noCache, + case Azure: + return NewAzureCache(noCache) + case FileBased: + return &FileBasedCache{ + noCache: noCache, + } + default: + return &FileBasedCache{ + noCache: noCache, + } } } // CacheProvider is the configuration for the cache provider when using a remote cache type CacheProvider struct { - BucketName string `mapstructure:"bucketname"` - Region string `mapstructure:"region"` + BucketName string `mapstructure:"bucketname" yaml:"bucketname,omitempty"` + Region string `mapstructure:"region" yaml:"region,omitempty"` + StorageAccount string `mapstructure:"storageaccount" yaml:"storageaccount,omitempty"` + ContainerName string `mapstructure:"container" yaml:"container,omitempty"` +} + +// NewCacheProvider constructs a new cache struct +func NewCacheProvider(bucketname, region, storageaccount, containername string) CacheProvider { + return CacheProvider{ + BucketName: bucketname, + Region: region, + StorageAccount: storageaccount, + ContainerName: containername, + } } -func RemoteCacheEnabled() (bool, error) { +// If we have set a remote cache, return the remote cache type +func RemoteCacheEnabled() (CacheType, error) { // load remote cache if it is configured var cache CacheProvider err := viper.UnmarshalKey("cache", &cache) if err != nil { - return false, err + return "", err } if cache.BucketName != "" && cache.Region != "" { - return true, nil + return S3, nil + } else if cache.StorageAccount != "" && cache.ContainerName != "" { + return Azure, nil } - return false, nil + return FileBased, nil } -func AddRemoteCache(bucketName string, region string) error { +func AddRemoteCache(cache CacheProvider) error { var cacheInfo CacheProvider err := viper.UnmarshalKey("cache", &cacheInfo) if err != nil { return err } - cacheInfo.BucketName = bucketName - cacheInfo.Region = region + cacheInfo.BucketName = cache.BucketName + cacheInfo.Region = cache.Region + cacheInfo.StorageAccount = cache.StorageAccount + cacheInfo.ContainerName = cache.ContainerName viper.Set("cache", cacheInfo) err = viper.WriteConfig() if err != nil { @@ -60,14 +93,14 @@ func AddRemoteCache(bucketName string, region string) error { return nil } -func RemoveRemoteCache(bucketName string) error { +func RemoveRemoteCache() error { var cacheInfo CacheProvider err := viper.UnmarshalKey("cache", &cacheInfo) if err != nil { return status.Error(codes.Internal, "cache unmarshal") } - if cacheInfo.BucketName == "" { - return status.Error(codes.Internal, "no cache configured") + if cacheInfo.BucketName == "" && cacheInfo.ContainerName == "" && cacheInfo.StorageAccount == "" { + return status.Error(codes.Internal, "no remote cache configured") } cacheInfo = CacheProvider{} diff --git a/pkg/cache/s3_based.go b/pkg/cache/s3_based.go index 9fa52a492d..444946ef83 100644 --- a/pkg/cache/s3_based.go +++ b/pkg/cache/s3_based.go @@ -2,6 +2,7 @@ package cache import ( "bytes" + "log" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -79,13 +80,13 @@ func NewS3Cache(nocache bool) ICache { var cache CacheProvider err := viper.UnmarshalKey("cache", &cache) if err != nil { - panic(err) + log.Fatal(err) } if cache.BucketName == "" { - panic("Bucket name not configured") + log.Fatal("Bucket name not configured") } if cache.Region == "" { - panic("Region not configured") + log.Fatal("Region not configured") } sess := session.Must(session.NewSessionWithOptions(session.Options{ diff --git a/pkg/server/config.go b/pkg/server/config.go index cd94c22229..26525c047e 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -1,8 +1,9 @@ package server import ( - schemav1 "buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go/schema/v1" "context" + + schemav1 "buf.build/gen/go/k8sgpt-ai/k8sgpt/protocolbuffers/go/schema/v1" "github.com/k8sgpt-ai/k8sgpt/pkg/cache" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -17,12 +18,14 @@ func (h *handler) AddConfig(ctx context.Context, i *schemav1.AddConfigRequest) ( } if i.Cache != nil { - // Remote cache - if i.Cache.BucketName == "" || i.Cache.Region == "" { - return resp, status.Error(codes.InvalidArgument, "cache arguments") + // We check if we have a mixed cache configuration + CacheConfigured := (i.Cache.Region == "" && i.Cache.BucketName == "") || (i.Cache.ContainerName == "" && i.Cache.StorageAccount == "") + if !CacheConfigured { + return resp, status.Error(codes.InvalidArgument, "mixed cache arguments") } - err := cache.AddRemoteCache(i.Cache.BucketName, i.Cache.Region) + cacheProvider := cache.NewCacheProvider(i.Cache.BucketName, i.Cache.Region, i.Cache.StorageAccount, i.Cache.ContainerName) + err := cache.AddRemoteCache(cacheProvider) if err != nil { return resp, err } @@ -32,7 +35,7 @@ func (h *handler) AddConfig(ctx context.Context, i *schemav1.AddConfigRequest) ( func (h *handler) RemoveConfig(ctx context.Context, i *schemav1.RemoveConfigRequest) (*schemav1.RemoveConfigResponse, error, ) { - err := cache.RemoveRemoteCache(i.Cache.BucketName) + err := cache.RemoveRemoteCache() if err != nil { return &schemav1.RemoveConfigResponse{}, err }