Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

r/glue_crawler - add support for scan_all and scan_rate arguments for ddb targets #14819

Merged
merged 4 commits into from
Sep 23, 2020
Merged

r/glue_crawler - add support for scan_all and scan_rate arguments for ddb targets #14819

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ea2c33c
add support for `scan_all` and `scan_rate` ddb target attributes
DrFaust92 Aug 25, 2020
65658f9
add test for scan all
DrFaust92 Aug 25, 2020
dbc7fba
add test for scan rate
DrFaust92 Aug 25, 2020
c7ecdf4
add docs for ddb target arguments
DrFaust92 Aug 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion aws/resource_aws_glue_crawler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,16 @@ func resourceAwsGlueCrawler() *schema.Resource {
Type: schema.TypeString,
Required: true,
},
"scan_all": {
Type: schema.TypeBool,
Optional: true,
Default: true,
},
"scan_rate": {
Type: schema.TypeFloat,
Optional: true,
ValidateFunc: validation.FloatBetween(0.1, 1.5),
},
},
},
},
Expand Down Expand Up @@ -373,7 +383,12 @@ func expandGlueDynamoDBTargets(targets []interface{}) []*glue.DynamoDBTarget {

func expandGlueDynamoDBTarget(cfg map[string]interface{}) *glue.DynamoDBTarget {
target := &glue.DynamoDBTarget{
Path: aws.String(cfg["path"].(string)),
Path: aws.String(cfg["path"].(string)),
ScanAll: aws.Bool(cfg["scan_all"].(bool)),
}

if v, ok := cfg["scan_rate"].(float64); ok && v != 0 {
target.ScanRate = aws.Float64(v)
}

return target
Expand Down Expand Up @@ -615,6 +630,8 @@ func flattenGlueDynamoDBTargets(dynamodbTargets []*glue.DynamoDBTarget) []map[st
for _, dynamodbTarget := range dynamodbTargets {
attrs := make(map[string]interface{})
attrs["path"] = aws.StringValue(dynamodbTarget.Path)
attrs["scan_all"] = aws.BoolValue(dynamodbTarget.ScanAll)
attrs["scan_rate"] = aws.Float64Value(dynamodbTarget.ScanRate)

result = append(result, attrs)
}
Expand Down
138 changes: 134 additions & 4 deletions aws/resource_aws_glue_crawler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ func TestAccAWSGlueCrawler_DynamodbTarget(t *testing.T) {
resource.TestCheckResourceAttr(resourceName, "description", ""),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.#", "1"),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_all", "true"),
resource.TestCheckResourceAttr(resourceName, "jdbc_target.#", "0"),
resource.TestCheckResourceAttr(resourceName, "name", rName),
resource.TestCheckResourceAttr(resourceName, "role", rName),
Expand All @@ -103,6 +104,7 @@ func TestAccAWSGlueCrawler_DynamodbTarget(t *testing.T) {
resource.TestCheckResourceAttr(resourceName, "description", ""),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.#", "1"),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table2"),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_all", "true"),
resource.TestCheckResourceAttr(resourceName, "jdbc_target.#", "0"),
resource.TestCheckResourceAttr(resourceName, "name", rName),
resource.TestCheckResourceAttr(resourceName, "role", rName),
Expand All @@ -124,6 +126,92 @@ func TestAccAWSGlueCrawler_DynamodbTarget(t *testing.T) {
})
}

func TestAccAWSGlueCrawler_DynamodbTarget_scanAll(t *testing.T) {
var crawler glue.Crawler
rName := acctest.RandomWithPrefix("tf-acc-test")
resourceName := "aws_glue_crawler.test"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSGlueCrawlerDestroy,
Steps: []resource.TestStep{
{
Config: testAccGlueCrawlerConfig_DynamodbTargetScanAll(rName, "table1", false),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueCrawlerExists(resourceName, &crawler),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_all", "false"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccGlueCrawlerConfig_DynamodbTargetScanAll(rName, "table1", true),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueCrawlerExists(resourceName, &crawler),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_all", "true"),
),
},
{
Config: testAccGlueCrawlerConfig_DynamodbTargetScanAll(rName, "table1", false),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueCrawlerExists(resourceName, &crawler),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_all", "false"),
),
},
},
})
}

func TestAccAWSGlueCrawler_DynamodbTarget_scanRate(t *testing.T) {
var crawler glue.Crawler
rName := acctest.RandomWithPrefix("tf-acc-test")
resourceName := "aws_glue_crawler.test"

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSGlueCrawlerDestroy,
Steps: []resource.TestStep{
{
Config: testAccGlueCrawlerConfig_DynamodbTargetScanRate(rName, "table1", 0.5),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueCrawlerExists(resourceName, &crawler),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_rate", "0.5"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
},
{
Config: testAccGlueCrawlerConfig_DynamodbTargetScanRate(rName, "table1", 1.5),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueCrawlerExists(resourceName, &crawler),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_rate", "1.5"),
),
},
{
Config: testAccGlueCrawlerConfig_DynamodbTargetScanRate(rName, "table1", 0.5),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSGlueCrawlerExists(resourceName, &crawler),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.path", "table1"),
resource.TestCheckResourceAttr(resourceName, "dynamodb_target.0.scan_rate", "0.5"),
),
},
},
})
}

func TestAccAWSGlueCrawler_JdbcTarget(t *testing.T) {
var crawler glue.Crawler
rName := acctest.RandomWithPrefix("tf-acc-test")
Expand Down Expand Up @@ -1211,21 +1299,63 @@ resource "aws_glue_crawler" "test" {
func testAccGlueCrawlerConfig_DynamodbTarget(rName, path string) string {
return testAccGlueCrawlerConfig_Base(rName) + fmt.Sprintf(`
resource "aws_glue_catalog_database" "test" {
name = %q
name = %[1]q
}

resource "aws_glue_crawler" "test" {
depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole]

database_name = aws_glue_catalog_database.test.name
name = %q
name = %[1]q
role = aws_iam_role.test.name

dynamodb_target {
path = %q
path = %[2]q
}
}
`, rName, rName, path)
`, rName, path)
}

func testAccGlueCrawlerConfig_DynamodbTargetScanAll(rName, path string, scanAll bool) string {
return testAccGlueCrawlerConfig_Base(rName) + fmt.Sprintf(`
resource "aws_glue_catalog_database" "test" {
name = %[1]q
}

resource "aws_glue_crawler" "test" {
depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole]

database_name = aws_glue_catalog_database.test.name
name = %[1]q
role = aws_iam_role.test.name

dynamodb_target {
path = %[2]q
scan_all = %[3]t
}
}
`, rName, path, scanAll)
}

func testAccGlueCrawlerConfig_DynamodbTargetScanRate(rName, path string, scanRate float64) string {
return testAccGlueCrawlerConfig_Base(rName) + fmt.Sprintf(`
resource "aws_glue_catalog_database" "test" {
name = %[1]q
}

resource "aws_glue_crawler" "test" {
depends_on = [aws_iam_role_policy_attachment.test-AWSGlueServiceRole]

database_name = aws_glue_catalog_database.test.name
name = %[1]q
role = aws_iam_role.test.name

dynamodb_target {
path = %[2]q
scan_rate = %[3]g
}
}
`, rName, path, scanRate)
}

func testAccGlueCrawlerConfig_JdbcTarget(rName, path string) string {
Expand Down
2 changes: 2 additions & 0 deletions website/docs/r/glue_crawler.html.markdown
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ The following arguments are supported:
### dynamodb_target Argument Reference

* `path` - (Required) The name of the DynamoDB table to crawl.
* `scan_all` - (Optional) Indicates whether to scan all the records, or to sample rows from the table. Scanning all the records can take a long time when the table is not a high throughput table. defaults to `true`.
* `scan_rate` - (Optional) The percentage of the configured read capacity units to use by the AWS Glue crawler. The valid values are null or a value between 0.1 to 1.5.

### jdbc_target Argument Reference

Expand Down