From f64a42b27177ff578504127bbcf72ac9340342cf Mon Sep 17 00:00:00 2001 From: Erik K Anderson Date: Fri, 25 Aug 2023 07:29:46 -0500 Subject: [PATCH 1/4] add redshift event subscription (wip) This is a wip commit with some todo question items. I would like to do a rebase before opening the PR, so that I send in one clean commit to the pull request. Also, there is another todo item: I should make the needed changes to go.mod, but no other changes. --- providers/aws/aws.go | 2 + providers/aws/redshift/eventsubscription.go | 87 +++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 providers/aws/redshift/eventsubscription.go diff --git a/providers/aws/aws.go b/providers/aws/aws.go index bab2af836..f7e77dd27 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -24,6 +24,7 @@ import ( "github.com/tailwarden/komiser/providers/aws/lambda" "github.com/tailwarden/komiser/providers/aws/opensearch" "github.com/tailwarden/komiser/providers/aws/rds" + "github.com/tailwarden/komiser/providers/aws/redshift" "github.com/tailwarden/komiser/providers/aws/s3" "github.com/tailwarden/komiser/providers/aws/servicecatalog" "github.com/tailwarden/komiser/providers/aws/sns" @@ -88,6 +89,7 @@ func listOfSupportedServices() []providers.FetchDataFunction { ec2.VpcEndpoints, ec2.VpcPeeringConnections, kinesis.Streams, + redshift.EventSubscription, } } diff --git a/providers/aws/redshift/eventsubscription.go b/providers/aws/redshift/eventsubscription.go new file mode 100644 index 000000000..cac40a7c4 --- /dev/null +++ b/providers/aws/redshift/eventsubscription.go @@ -0,0 +1,87 @@ +package redshift + +import ( + "context" + "fmt" + "time" + + log "github.com/sirupsen/logrus" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/redshift" + "github.com/aws/aws-sdk-go-v2/service/sts" + . "github.com/tailwarden/komiser/models" + . "github.com/tailwarden/komiser/providers" +) + +func EventSubscription(ctx context.Context, client ProviderClient) ([]Resource, error) { + resources := make([]Resource, 0) + var config redshift.DescribeEventSubscriptionsInput + redshiftClient := redshift.NewFromConfig(*client.AWSClient) + + stsClient := sts.NewFromConfig(*client.AWSClient) + stsOutput, err := stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}) + if err != nil { + return resources, err + } + + accountId := stsOutput.Account + + for { + output, err := redshiftClient.DescribeEventSubscriptions(ctx, &config) + if err != nil { + return resources, err + } + + for _, eventSubscription := range output.EventSubscriptionsList { // TODO: capitalization convention? + if eventSubscription.CustSubscriptionId != nil { // TODO: is this equivalent to filesystem.Name from the efs example? + + resourceArn := fmt.Sprintf("arn:aws:redshift:%s:%s:eventsubscripion/%s", client.AWSClient.Region, *accountId, *eventSubscription.CustSubscriptionId) // TODO: is this arn format correct + outputTags, err := redshiftClient.DescribeTags(ctx, &redshift.DescribeTagsInput{ + ResourceName: &resourceArn, // TODO: is ResourceName here equivalent to ResourceId in the efs example? + }) + + tags := make([]Tag, 0) + + if err == nil { + for _, tag := range outputTags.TaggedResources { // TODO: this is slightly different than in the efs example. Is it correct? + tags = append(tags, Tag{ + Key: *tag.Tag.Key, + Value: *tag.Tag.Value, + }) + } + } + + monthlyCost := float64(0) // TODO: what is the monthly cost + + resources = append(resources, Resource{ + Provider: "AWS", + Account: client.Name, + Service: "Redshift", + ResourceId: resourceArn, + Region: client.AWSClient.Region, + Name: *eventSubscription.CustSubscriptionId, + Cost: monthlyCost, + Tags: tags, + FetchedAt: time.Now(), + Link: fmt.Sprintf("https://%s.console.aws.amaxon.com/redshift/home?region=%s/event-subscriptions/%s", client.AWSClient.Region, client.AWSClient.Region, eventSubscription.CustSubscriptionId), // TODO: verify that the link format is correct + }) + } + } + + if aws.ToString(output.Marker) == "" { // TODO: is output.Marker here playing the same role as ouput.NextMarker in the efs example? + break + } + config.Marker = output.Marker + } + + log.WithFields(log.Fields{ + "provider": "AWS", + "account": client.Name, + "region": client.AWSClient.Region, + "service": "Redshift", + "resources": len(resources), + }).Info("Fetched resources") + return resources, nil + +} From 885e9216ea6db58e81bb049bc60d7ec1a236445a Mon Sep 17 00:00:00 2001 From: Erik K Anderson Date: Mon, 28 Aug 2023 17:35:24 -0500 Subject: [PATCH 2/4] use go get github.com/aws/aws-sdk-go-v2/service/redshift --- go.mod | 9 +++++---- go.sum | 10 ++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index c74592f9c..ae88e7534 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/sql/armsql v1.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 github.com/BurntSushi/toml v1.2.1 - github.com/aws/aws-sdk-go-v2 v1.18.1 + github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/config v1.18.27 github.com/aws/aws-sdk-go-v2/service/apigateway v1.16.0 github.com/aws/aws-sdk-go-v2/service/autoscaling v1.27.2 @@ -81,6 +81,7 @@ require ( require ( cloud.google.com/go/longrunning v0.4.1 // indirect github.com/apache/arrow/go/v11 v11.0.0 // indirect + github.com/aws/aws-sdk-go-v2/service/redshift v1.29.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect ) @@ -97,8 +98,8 @@ require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.5 // indirect github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.20.9 @@ -111,7 +112,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/servicecatalog v1.18.2 github.com/aws/aws-sdk-go-v2/service/ssm v1.36.2 github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect - github.com/aws/smithy-go v1.13.5 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect diff --git a/go.sum b/go.sum index eb8865a36..b2023ae3f 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,8 @@ github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo= github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= +github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.3/go.mod h1:gNsR5CaXKmQSSzrmGxmwmct/r+ZBfbxorAuXYsj/M5Y= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 h1:dK82zF6kkPeCo8J1e+tGx4JdvDIQzj7ygIoLg8WMuGs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= @@ -90,6 +92,8 @@ github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11X github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.8/go.mod h1:ZIV8GYoC6WLBW5KGs+o4rsc65/ozd+eQ0L31XF5VDwk= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= @@ -100,6 +104,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7im github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15 h1:QquxR7NH3ULBsKC+NoTpilzbKKS+5AELfNREInbhvas= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.15/go.mod h1:Tkrthp/0sNBShQQsamR7j/zY4p19tVTAs+nnqhH6R3c= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= @@ -164,6 +170,8 @@ github.com/aws/aws-sdk-go-v2/service/pricing v1.20.0 h1:x5gKeerbKIQ/tdhmaAGNpivS github.com/aws/aws-sdk-go-v2/service/pricing v1.20.0/go.mod h1:JjpnqJdEW/5An429Ou+5Kb3UkwjXv16gRD2ZdGA2Gw8= github.com/aws/aws-sdk-go-v2/service/rds v1.30.1 h1:/B3GswjV+ScqZSZnhs3NMLwvVFXb1/aQL/elQwo0CMM= github.com/aws/aws-sdk-go-v2/service/rds v1.30.1/go.mod h1:wPFe1Cj3nZWmNWKKdkXw961l1dJheTZQ5JjPImqbMuI= +github.com/aws/aws-sdk-go-v2/service/redshift v1.29.5 h1:ufl4QI+6Vuxg6E8UOFVy+CeCtXS+gBMb00oTh2qSPco= +github.com/aws/aws-sdk-go-v2/service/redshift v1.29.5/go.mod h1:U8V+thdAH44/2weiprIA0JyDWa2XBov58TtdjCTTpc8= github.com/aws/aws-sdk-go-v2/service/s3 v1.27.1 h1:OKQIQ0QhEBmGr2LfT952meIZz3ujrPYnxH+dO/5ldnI= github.com/aws/aws-sdk-go-v2/service/s3 v1.27.1/go.mod h1:NffjpNsMUFXp6Ok/PahrktAncoekWrywvmIK83Q2raE= github.com/aws/aws-sdk-go-v2/service/servicecatalog v1.18.2 h1:YQ7/6UE9h6dIHMjHFr4xXLMZDfhwtQUea/D18XAjhcE= @@ -188,6 +196,8 @@ github.com/aws/smithy-go v1.12.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= +github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= From 6b167b809ac4da7bcb736215cb132783f51a5ac9 Mon Sep 17 00:00:00 2001 From: Erik K Anderson Date: Wed, 30 Aug 2023 15:54:01 -0500 Subject: [PATCH 3/4] implement suggestions from pull request conversation --- providers/aws/aws.go | 2 +- providers/aws/redshift/eventsubscription.go | 24 ++++++++++----------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/providers/aws/aws.go b/providers/aws/aws.go index 89c40c467..dfb0931ff 100644 --- a/providers/aws/aws.go +++ b/providers/aws/aws.go @@ -90,7 +90,7 @@ func listOfSupportedServices() []providers.FetchDataFunction { ec2.VpcEndpoints, ec2.VpcPeeringConnections, kinesis.Streams, - redshift.EventSubscription, + redshift.EventSubscriptions, } } diff --git a/providers/aws/redshift/eventsubscription.go b/providers/aws/redshift/eventsubscription.go index cac40a7c4..40d076199 100644 --- a/providers/aws/redshift/eventsubscription.go +++ b/providers/aws/redshift/eventsubscription.go @@ -14,7 +14,7 @@ import ( . "github.com/tailwarden/komiser/providers" ) -func EventSubscription(ctx context.Context, client ProviderClient) ([]Resource, error) { +func EventSubscriptions(ctx context.Context, client ProviderClient) ([]Resource, error) { resources := make([]Resource, 0) var config redshift.DescribeEventSubscriptionsInput redshiftClient := redshift.NewFromConfig(*client.AWSClient) @@ -33,21 +33,19 @@ func EventSubscription(ctx context.Context, client ProviderClient) ([]Resource, return resources, err } - for _, eventSubscription := range output.EventSubscriptionsList { // TODO: capitalization convention? - if eventSubscription.CustSubscriptionId != nil { // TODO: is this equivalent to filesystem.Name from the efs example? + for _, eventSubscription := range output.EventSubscriptionsList { + if eventSubscription.CustSubscriptionId != nil { resourceArn := fmt.Sprintf("arn:aws:redshift:%s:%s:eventsubscripion/%s", client.AWSClient.Region, *accountId, *eventSubscription.CustSubscriptionId) // TODO: is this arn format correct - outputTags, err := redshiftClient.DescribeTags(ctx, &redshift.DescribeTagsInput{ - ResourceName: &resourceArn, // TODO: is ResourceName here equivalent to ResourceId in the efs example? - }) + outputTags := eventSubscription.Tags tags := make([]Tag, 0) if err == nil { - for _, tag := range outputTags.TaggedResources { // TODO: this is slightly different than in the efs example. Is it correct? + for _, tag := range outputTags { tags = append(tags, Tag{ - Key: *tag.Tag.Key, - Value: *tag.Tag.Value, + Key: *tag.Key, + Value: *tag.Value, }) } } @@ -57,19 +55,19 @@ func EventSubscription(ctx context.Context, client ProviderClient) ([]Resource, resources = append(resources, Resource{ Provider: "AWS", Account: client.Name, - Service: "Redshift", + Service: "Redshift EventSubscription", ResourceId: resourceArn, Region: client.AWSClient.Region, Name: *eventSubscription.CustSubscriptionId, Cost: monthlyCost, Tags: tags, FetchedAt: time.Now(), - Link: fmt.Sprintf("https://%s.console.aws.amaxon.com/redshift/home?region=%s/event-subscriptions/%s", client.AWSClient.Region, client.AWSClient.Region, eventSubscription.CustSubscriptionId), // TODO: verify that the link format is correct + Link: fmt.Sprintf("https://%s.console.aws.amaxon.com/redshift/home?region=%s/event-subscriptions/%s", client.AWSClient.Region, client.AWSClient.Region, *eventSubscription.CustSubscriptionId), // TODO: verify that the link format is correct }) } } - if aws.ToString(output.Marker) == "" { // TODO: is output.Marker here playing the same role as ouput.NextMarker in the efs example? + if aws.ToString(output.Marker) == "" { break } config.Marker = output.Marker @@ -79,7 +77,7 @@ func EventSubscription(ctx context.Context, client ProviderClient) ([]Resource, "provider": "AWS", "account": client.Name, "region": client.AWSClient.Region, - "service": "Redshift", + "service": "Redshift EventSubscription", "resources": len(resources), }).Info("Fetched resources") return resources, nil From a81c33fdc85cddee917cbe0358231eb7976e3b67 Mon Sep 17 00:00:00 2001 From: Erik K Anderson Date: Fri, 1 Sep 2023 04:04:21 -0500 Subject: [PATCH 4/4] clear out todo comments --- providers/aws/redshift/eventsubscription.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/providers/aws/redshift/eventsubscription.go b/providers/aws/redshift/eventsubscription.go index 40d076199..d2b488d8c 100644 --- a/providers/aws/redshift/eventsubscription.go +++ b/providers/aws/redshift/eventsubscription.go @@ -36,7 +36,7 @@ func EventSubscriptions(ctx context.Context, client ProviderClient) ([]Resource, for _, eventSubscription := range output.EventSubscriptionsList { if eventSubscription.CustSubscriptionId != nil { - resourceArn := fmt.Sprintf("arn:aws:redshift:%s:%s:eventsubscripion/%s", client.AWSClient.Region, *accountId, *eventSubscription.CustSubscriptionId) // TODO: is this arn format correct + resourceArn := fmt.Sprintf("arn:aws:redshift:%s:%s:eventsubscripion/%s", client.AWSClient.Region, *accountId, *eventSubscription.CustSubscriptionId) outputTags := eventSubscription.Tags tags := make([]Tag, 0) @@ -50,7 +50,7 @@ func EventSubscriptions(ctx context.Context, client ProviderClient) ([]Resource, } } - monthlyCost := float64(0) // TODO: what is the monthly cost + monthlyCost := float64(0) resources = append(resources, Resource{ Provider: "AWS", @@ -62,7 +62,7 @@ func EventSubscriptions(ctx context.Context, client ProviderClient) ([]Resource, Cost: monthlyCost, Tags: tags, FetchedAt: time.Now(), - Link: fmt.Sprintf("https://%s.console.aws.amaxon.com/redshift/home?region=%s/event-subscriptions/%s", client.AWSClient.Region, client.AWSClient.Region, *eventSubscription.CustSubscriptionId), // TODO: verify that the link format is correct + Link: fmt.Sprintf("https://%s.console.aws.amaxon.com/redshift/home?region=%s/event-subscriptions/%s", client.AWSClient.Region, client.AWSClient.Region, *eventSubscription.CustSubscriptionId), }) } }