feat: make crawler optional in GlueTransformStage() (#378)

* enhancment: make crawler optional * chore: self mutation Signed-off-by: github-actions <github-actions@github.com> * reverting changes to workflow --------- Signed-off-by: github-actions <github-actions@github.com> Co-authored-by: github-actions <github-actions@github.com>
awslabs · Jul 3, 2023 · 49084a8 · 49084a8
1 parent 1f5663e
commit 49084a8
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 34 deletions.
diff --git a/API.md b/API.md
diff --git a/src/stages/glue-transform.ts b/src/stages/glue-transform.ts
@@ -78,6 +78,8 @@ export class GlueTransformStage extends StateMachineStage {
   readonly stateMachine: sfn.StateMachine;
   readonly glueJob: glue_alpha.IJob;
   readonly crawler?: glue.CfnCrawler;
+  readonly crawlerName?: string;
+  readonly definition: sfn.IChainable;
 
   /**
    * Constructs `GlueTransformStage`.
@@ -91,8 +93,11 @@ export class GlueTransformStage extends StateMachineStage {
     this.glueJob = this.getGlueJob(scope, id, props);
     const jobRunArgs = props.jobRunArgs;
 
-    this.crawler = props.crawlerName ? undefined : this.getCrawler(props);
-    const crawlerName = this.crawler ? this.crawler.ref : props.crawlerName;
+    // Only setup crawler if one of 'crawlerName', 'crawlerProps' or 'crawlerRole' is provided
+    if (props.crawlerName || props.crawlerProps || props.crawlerRole) {
+      this.crawler = props.crawlerName ? undefined : this.getCrawler(props);
+      this.crawlerName = this.crawler ? this.crawler.ref : props.crawlerName;
+    }
 
     const startJobRun = new tasks.GlueStartJobRun(this, "Start Job Run", {
       glueJobName: this.glueJob.jobName,
@@ -102,36 +107,37 @@ export class GlueTransformStage extends StateMachineStage {
     });
 
     const stack = cdk.Stack.of(this);
-    const crawlObject = new tasks.CallAwsService(this, "Crawl Object", {
-      service: "glue",
-      action: "startCrawler",
-      parameters: {
-        Name: crawlerName,
-      },
-      iamResources: [`arn:${stack.partition}:glue:${stack.region}:${stack.account}:crawler/${crawlerName}`],
-    });
-
     const successTask = new sfn.Succeed(this, "Success");
 
-    crawlObject.addRetry({
-      errors: ["Glue.CrawlerRunningException"],
-      maxAttempts: props.stateMachineRetryMaxAttempts ?? 3,
-      backoffRate: props.stateMachineRetryBackoffRate ?? 2,
-      interval: props.stateMachineRetryInterval ?? cdk.Duration.seconds(1),
-    });
-
-    const crawlerAllowFailure = props.crawlerAllowFailure ?? true;
-    if (crawlerAllowFailure) {
-      crawlObject.addCatch(successTask, { errors: ["Glue.CrawlerRunningException"] });
+    if (this.crawlerName) {
+      const crawlObject = new tasks.CallAwsService(this, "Crawl Object", {
+        service: "glue",
+        action: "startCrawler",
+        parameters: {
+          Name: this.crawlerName,
+        },
+        iamResources: [`arn:${stack.partition}:glue:${stack.region}:${stack.account}:crawler/${this.crawlerName}`],
+      });
+      crawlObject.addRetry({
+        errors: ["Glue.CrawlerRunningException"],
+        maxAttempts: props.stateMachineRetryMaxAttempts ?? 3,
+        backoffRate: props.stateMachineRetryBackoffRate ?? 2,
+        interval: props.stateMachineRetryInterval ?? cdk.Duration.seconds(1),
+      });
+      const crawlerAllowFailure = props.crawlerAllowFailure ?? true;
+      if (crawlerAllowFailure) {
+        crawlObject.addCatch(successTask, { errors: ["Glue.CrawlerRunningException"] });
+      }
+      this.definition = startJobRun.next(crawlObject.next(successTask));
+    } else {
+      this.definition = startJobRun.next(successTask);
     }
 
-    const definition = startJobRun.next(crawlObject.next(successTask));
-
     ({
       eventPattern: this.eventPattern,
       targets: this.targets,
       stateMachine: this.stateMachine,
-    } = this.createStateMachine({ definition: definition, ...props }));
+    } = this.createStateMachine({ definition: this.definition, ...props }));
   }
 
   private getGlueJob(scope: Construct, id: string, props: GlueTransformStageProps): glue_alpha.IJob {

diff --git a/test/glue-transform-stage.test.ts b/test/glue-transform-stage.test.ts
@@ -107,15 +107,6 @@ test("GlueTranformStage must have 'jobName' or 'jobProps' set", () => {
   }).toThrowError("'jobName' or 'jobProps' must be set to instantiate this stage");
 });
 
-test("GlueTranformStage must set crawler role ", () => {
-  const stack = new cdk.Stack();
-  expect(() => {
-    new GlueTransformStage(stack, "Stage", {
-      jobName: "myJob",
-    });
-  }).toThrowError("Crawler Role must be set either by 'crawlerRole' or 'crawlerProps.role");
-});
-
 test("GlueTranformStage must set crawler targets", () => {
   const stack = new cdk.Stack();
   expect(() => {
@@ -148,7 +139,7 @@ test("GlueTransformStage retry settings", () => {
   });
 });
 
-test("GlueTransformStage crawler allo failure settings", () => {
+test("GlueTransformStage crawler allow failure settings", () => {
   const stack = new cdk.Stack();
 
   new GlueTransformStage(stack, "glue-transform-disallow-failure", {
@@ -169,3 +160,14 @@ test("GlueTransformStage crawler allo failure settings", () => {
     },
   });
 });
+
+test("GlueTransformStage no crawler", () => {
+  const stack = new cdk.Stack();
+
+  new GlueTransformStage(stack, "glue-transform-no-crawler", {
+    jobName: "myJob",
+  });
+
+  const template = Template.fromStack(stack);
+  template.resourceCountIs("AWS::Glue::Crawler", 0);
+});