Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Kafka support to IOT topic rules #24395

Merged
merged 90 commits into from
May 5, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
90 commits
Select commit Hold shift + click to select a range
79edd37
added support for iot rule action "http"
dbasedow Nov 8, 2020
a120e2c
rename headers to http_header
dbasedow Nov 9, 2020
72a156d
add support for http error actions
dbasedow Nov 9, 2020
796742f
update documentation for http action
dbasedow Nov 9, 2020
d4767b9
fix formatting of test data
dbasedow Nov 9, 2020
76dae68
add kafka rule type to iot topic rules
toddradigan Apr 7, 2021
e06d81f
add changelog
toddradigan Apr 7, 2021
d4712e1
escape get_secret
toddradigan Apr 8, 2021
75db504
randomize secret name in test
toddradigan Apr 8, 2021
4716b20
lint: TypeSet -> TypeList
toddradigan Apr 8, 2021
8b3392f
lint: fix spacing
toddradigan Apr 8, 2021
5f837a9
remove secret creation
toddradigan Apr 8, 2021
45c5f7d
r/aws_iot_topic_rule: read support for timestream action
thiagoarrais Dec 22, 2021
d5b741b
r/aws_iot_topic_rule: create/update support for timestream action
thiagoarrais Dec 22, 2021
dccefb9
r/aws_iot_topic_rule: timestream as error action
thiagoarrais Dec 23, 2021
37cc161
r/aws_iot_topic_rule: test for timestream as iot action
thiagoarrais Dec 23, 2021
6f0e727
r/aws_iot_topic_rule: docs for timestream as iot action
thiagoarrais Dec 23, 2021
0c398c2
initial push
Apr 22, 2022
4d50169
adding tests
Apr 25, 2022
f2d0d7d
adding changelog
Apr 25, 2022
f2d578d
Revert "remove secret creation"
ewbankkit May 3, 2022
d90a7f5
Revert "lint: fix spacing"
ewbankkit May 3, 2022
09944a8
Revert "lint: TypeSet -> TypeList"
ewbankkit May 3, 2022
276f511
Revert "randomize secret name in test"
ewbankkit May 3, 2022
a28a6dc
Revert "escape get_secret"
ewbankkit May 3, 2022
7cf6c37
Revert "add changelog"
ewbankkit May 3, 2022
e8db406
Revert "add kafka rule type to iot topic rules"
ewbankkit May 3, 2022
9b2454f
Merge commit 'e8db406ea6a3840efa5509a4facbb3f0b7f13c57' into HEAD
ewbankkit May 3, 2022
7e80260
Merge remote-tracking branch 'thiagoarrais/timestream-iot-action' int…
ewbankkit May 3, 2022
d58bec4
Revert "fix formatting of test data"
ewbankkit May 3, 2022
4db34c1
Revert "update documentation for http action"
ewbankkit May 3, 2022
7aab3bb
Revert "add support for http error actions"
ewbankkit May 3, 2022
8230107
Revert "rename headers to http_header"
ewbankkit May 3, 2022
c8bb3fa
Revert "added support for iot rule action "http""
ewbankkit May 3, 2022
671fd60
Merge commit 'c8bb3fa280710ab7757952f77b31ec3efe8ce984' into HEAD
ewbankkit May 3, 2022
7746d5f
Add CHANGELOG entry for #16087.
ewbankkit May 3, 2022
c312f92
Add CHANGELOG entry for #22337.
ewbankkit May 3, 2022
e4d77e1
Tweak CHANGELOG entry for #24395.
ewbankkit May 3, 2022
a3e64aa
r/aws_iot_topic_rule: Alphabetize attributes.
ewbankkit May 3, 2022
25d6423
r/aws_iot_topic_rule: Add and use 'topicRuleErrorActionExactlyOneOf'.
ewbankkit May 3, 2022
4115831
More consistent order of expanders.
ewbankkit May 3, 2022
c7b1a81
More consistent order of flatteners.
ewbankkit May 3, 2022
e134af2
More consistent order of acceptance tests.
ewbankkit May 3, 2022
fd50f26
r/aws_iot_topic_rule: Add and use 'FindTopicRuleByName'.
ewbankkit May 3, 2022
ccd2a2f
r/aws_iot_topic_rule: Tidy up resource Delete.
ewbankkit May 3, 2022
732e814
r/aws_iot_topic_rule: Tidy up resource Update.
ewbankkit May 3, 2022
556066a
r/aws_iot_topic_rule: Tidy up resource Create.
ewbankkit May 3, 2022
e1f2737
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_tags'.
ewbankkit May 3, 2022
3cf854c
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_cloudWatchAlarm'.
ewbankkit May 3, 2022
db4f751
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_cloudWatchLogs'.
ewbankkit May 3, 2022
8f5f9f4
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_cloudWatchMetric'.
ewbankkit May 3, 2022
4245c5b
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_dynamoDB'.
ewbankkit May 3, 2022
c0e9976
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_elasticSearch'.
ewbankkit May 3, 2022
c23f11a
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_firehose'.
ewbankkit May 3, 2022
11a4709
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_Firehose_separator'.
ewbankkit May 3, 2022
b4fcc61
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_IoT_analytics'.
ewbankkit May 4, 2022
8e036fb
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_IoT_events'.
ewbankkit May 4, 2022
ae90793
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_kafka'.
ewbankkit May 4, 2022
4105e06
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_kinesis'.
ewbankkit May 4, 2022
1f259af
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_lambda'.
ewbankkit May 4, 2022
c623a03
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_republish'.
ewbankkit May 4, 2022
ef0091a
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_s3'.
ewbankkit May 4, 2022
8195d0c
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_sns'.
ewbankkit May 4, 2022
2f378bf
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_sqs'.
ewbankkit May 4, 2022
adf3f79
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_Step_functions'.
ewbankkit May 4, 2022
f8e0631
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_Timestream'.
ewbankkit May 4, 2022
fd0a964
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_errorAction'.
ewbankkit May 4, 2022
77ef936
r/aws_iot_topic_rule: Tidy up 'TestAccIoTTopicRule_updateKinesisError…
ewbankkit May 4, 2022
14edd18
'aws_iot_topic_rule.rule' -> 'aws_iot_topic_rule.test'.
ewbankkit May 4, 2022
2b8b7ff
'aws_iam_role.iot_role.arn' -> 'aws_iam_role.test.arn'.
ewbankkit May 4, 2022
ae702ed
r/aws_iot_topic_rule: Tidy up documentation.
ewbankkit May 4, 2022
b149e37
r/aws_iot_topic_rule: Add 'http' and 'error_action.http'.
ewbankkit May 4, 2022
349c985
r/aws_iot_topic_rule: Add 'http' and 'error_action.http' to documenta…
ewbankkit May 4, 2022
84319d5
r/aws_iot_topic_rule: Add 'TestAccIoTTopicRule_http'.
ewbankkit May 4, 2022
909eb0f
r/aws_iot_topic_rule_destination: New resource.
ewbankkit May 4, 2022
d7ef866
r/aws_iot_topic_rule_destination: Start to add acceptance tests.
ewbankkit May 4, 2022
114c179
r/aws_iot_topic_rule_destination: Remove 'http_url_configuration' arg…
ewbankkit May 4, 2022
6f03891
r/aws_iot_topic_rule_destination: Add documentation.
ewbankkit May 4, 2022
0aa890c
r/aws_iot_topic_rule_destination: Add sweeper.
ewbankkit May 4, 2022
7702d3c
r/aws_iot_topic_rule_destination: Increase default deletion timeout.
ewbankkit May 4, 2022
9e6e0dc
'FindTopicRuleDestinationByARN' now returns a NotFoundError if the de…
ewbankkit May 4, 2022
4c60bad
r/aws_iot_topic_rule_destination: Allow 'enabled' to be updated.
ewbankkit May 4, 2022
64ad127
r/aws_iot_topic_rule_destination: Don't forget to actually call 'Upda…
ewbankkit May 5, 2022
86f2fe9
r/aws_topic_rule: Add 'kafka.client_properties' as TypeMap.
ewbankkit May 5, 2022
c7b0672
More checks in 'TestAccIoTTopicRule_kafka'.
ewbankkit May 5, 2022
bc48ba2
r/aws_iot_topic_rule: Add documentation for 'kafka'.
ewbankkit May 5, 2022
31ebacb
Fix terrafmt error.
ewbankkit May 5, 2022
841474f
Fix 'MD047/single-trailing-newline Files should end with a single new…
ewbankkit May 5, 2022
a3106f2
r/aws_iot_topic_rule: Add 's3.canned_acl'.
ewbankkit May 5, 2022
8b059a1
Skip tests for unsupported GovCloud functionality.
ewbankkit May 5, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .changelog/16087.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
resource/aws_iot_topic_rule: Add `http` and `error_action.http` arguments
```
3 changes: 3 additions & 0 deletions .changelog/19175.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
resource/aws_iot_topic_rule: Add `s3.canned_acl` and `error_action.s3.canned_acl` arguments
```
3 changes: 3 additions & 0 deletions .changelog/22337.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
resource/aws_iot_topic_rule: Add `timestream` and `error_action.timestream` arguments
```
7 changes: 7 additions & 0 deletions .changelog/24395.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
```release-note:enhancement
resource/aws_iot_topic_rule: Add `kafka` and `error_action.kafka` arguments
```

```release-note:new-resource
aws_iot_topic_rule_destination
```
1 change: 1 addition & 0 deletions internal/provider/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -1530,6 +1530,7 @@ func Provider() *schema.Provider {
"aws_iot_thing_principal_attachment": iot.ResourceThingPrincipalAttachment(),
"aws_iot_thing_type": iot.ResourceThingType(),
"aws_iot_topic_rule": iot.ResourceTopicRule(),
"aws_iot_topic_rule_destination": iot.ResourceTopicRuleDestination(),

"aws_msk_cluster": kafka.ResourceCluster(),
"aws_msk_configuration": kafka.ResourceConfiguration(),
Expand Down
1 change: 1 addition & 0 deletions internal/service/ec2/sweep.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ func init() {
"aws_fsx_ontap_file_system",
"aws_fsx_openzfs_file_system",
"aws_fsx_windows_file_system",
"aws_iot_topic_rule_destination",
"aws_lambda_function",
"aws_lb",
"aws_memorydb_subnet_group",
Expand Down
104 changes: 104 additions & 0 deletions internal/service/iot/find.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package iot

import (
"context"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/iot"
"github.com/hashicorp/aws-sdk-go-base/v2/awsv1shim/v2/tfawserr"
Expand Down Expand Up @@ -119,3 +121,105 @@ func FindThingGroupMembership(conn *iot.IoT, thingGroupName, thingName string) e

return nil
}

func FindTopicRuleByName(conn *iot.IoT, name string) (*iot.GetTopicRuleOutput, error) {
// GetTopicRule returns unhelpful errors such as
// "An error occurred (UnauthorizedException) when calling the GetTopicRule operation: Access to topic rule 'xxxxxxxx' was denied"
// when querying for a rule that doesn't exist.
var rule *iot.TopicRuleListItem

err := conn.ListTopicRulesPages(&iot.ListTopicRulesInput{}, func(page *iot.ListTopicRulesOutput, lastPage bool) bool {
if page == nil {
return !lastPage
}

for _, v := range page.Rules {
if v == nil {
continue
}

if aws.StringValue(v.RuleName) == name {
rule = v

return false
}
}

return !lastPage
})

if err != nil {
return nil, err
}

if rule == nil {
return nil, tfresource.NewEmptyResultError(name)
}

input := &iot.GetTopicRuleInput{
RuleName: aws.String(name),
}

output, err := conn.GetTopicRule(input)

if err != nil {
return nil, err
}

if output == nil {
return nil, tfresource.NewEmptyResultError(input)
}

return output, nil
}

func FindTopicRuleDestinationByARN(ctx context.Context, conn *iot.IoT, arn string) (*iot.TopicRuleDestination, error) {
// GetTopicRuleDestination returns unhelpful errors such as
// "UnauthorizedException: Access to TopicRuleDestination 'arn:aws:iot:us-west-2:123456789012:ruledestination/vpc/f267138a-7383-4670-9e44-a7fe2f48af5e' was denied"
// when querying for a rule destination that doesn't exist.
var destination *iot.TopicRuleDestinationSummary

err := conn.ListTopicRuleDestinationsPages(&iot.ListTopicRuleDestinationsInput{}, func(page *iot.ListTopicRuleDestinationsOutput, lastPage bool) bool {
if page == nil {
return !lastPage
}

for _, v := range page.DestinationSummaries {
if v == nil {
continue
}

if aws.StringValue(v.Arn) == arn {
destination = v

return false
}
}

return !lastPage
})

if err != nil {
return nil, err
}

if destination == nil {
return nil, tfresource.NewEmptyResultError(destination)
}

input := &iot.GetTopicRuleDestinationInput{
Arn: aws.String(arn),
}

output, err := conn.GetTopicRuleDestinationWithContext(ctx, input)

if err != nil {
return nil, err
}

if output == nil || output.TopicRuleDestination == nil {
return nil, tfresource.NewEmptyResultError(input)
}

return output.TopicRuleDestination, nil
}
53 changes: 51 additions & 2 deletions internal/service/iot/sweep.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,14 @@ func init() {
})

resource.AddTestSweepers("aws_iot_topic_rule", &resource.Sweeper{
Name: "aws_iot_topic_rule",
F: sweepTopicRules,
Name: "aws_iot_topic_rule",
F: sweepTopicRules,
Dependencies: []string{"aws_iot_topic_rule_destination"},
})

resource.AddTestSweepers("aws_iot_topic_rule_destination", &resource.Sweeper{
Name: "aws_iot_topic_rule_destination",
F: sweepTopicRuleDestinations,
})
}

Expand Down Expand Up @@ -523,3 +529,46 @@ func sweepThingGroups(region string) error {

return nil
}

func sweepTopicRuleDestinations(region string) error {
client, err := sweep.SharedRegionalSweepClient(region)
if err != nil {
return fmt.Errorf("error getting client: %w", err)
}
conn := client.(*conns.AWSClient).IoTConn
input := &iot.ListTopicRuleDestinationsInput{}
sweepResources := make([]*sweep.SweepResource, 0)

err = conn.ListTopicRuleDestinationsPages(input, func(page *iot.ListTopicRuleDestinationsOutput, lastPage bool) bool {
if page == nil {
return !lastPage
}

for _, v := range page.DestinationSummaries {
r := ResourceTopicRuleDestination()
d := r.Data(nil)
d.SetId(aws.StringValue(v.Arn))

sweepResources = append(sweepResources, sweep.NewSweepResource(r, d, client))
}

return !lastPage
})

if sweep.SkipSweepError(err) {
log.Printf("[WARN] Skipping IoT Topic Rule Destination sweep for %s: %s", region, err)
return nil
}

if err != nil {
return fmt.Errorf("error listing IoT Topic Rule Destinations (%s): %w", region, err)
}

err = sweep.SweepOrchestrator(sweepResources)

if err != nil {
return fmt.Errorf("error sweeping IoT Topic Rule Destinations (%s): %w", region, err)
}

return nil
}
Loading