Initiate CDK Pipeline

.gitignore

@@ -53,3 +53,6 @@ cdk.context.json
 data/
 
 Brewfile.lock.json
+*.xml
+
+target/

Makefile

@@ -23,11 +23,17 @@ baseline:
 test:
 	@yarn test
 
+# Test only section 
+test-stateful:
+	@yarn run test ./test/stateful
+test-stateless:
+	@yarn run test ./test/stateless
+
 # Run all test suites - i.e. cdk app unit tests + each microservice app test suites
 # Each app root should have Makefile `test` target; that run your app test pipeline including compose stack up/down
 # Note by running `make suite` target from repo root means your local dev env is okay with all app toolchains i.e.
 # 	Python (conda or venv), Rust and Cargo, TypeScript and Node environment, Docker and Container runtimes
-suite: test
+suite: test-stateless
 	@(cd lib/workload/stateless/sequence_run_manager && $(MAKE) test)
 	@(cd lib/workload/stateless/metadata_manager && $(MAKE) test)
 	@#(cd lib/workload/stateless/filemanager && $(MAKE) test)   # FIXME uncomment when ready @Marko

bin/orcabus.ts

@@ -15,8 +15,8 @@ import { getEnvironmentConfig } from '../config/constants';
 const app = new cdk.App();
 const props: cdk.StackProps = {
   env: {
-    account: process.env.CDK_DEFAULT_ACCOUNT,
-    region: process.env.CDK_DEFAULT_REGION,
+    account: process.env.CDK_DEPLOY_ACCOUNT || process.env.CDK_DEFAULT_ACCOUNT,
+    region: process.env.CDK_DEPLOY_REGION || process.env.CDK_DEFAULT_REGION,
   },
   tags: {
     'umccr-org:Stack': 'OrcaBusSandboxApp',

bin/pipeline.ts → bin/stateful-pipeline.ts

@@ -2,20 +2,20 @@
 import 'source-map-support/register';
 
 import * as cdk from 'aws-cdk-lib';
-import { PipelineStack } from '../lib/pipeline/orcabus-pipeline-stack';
+import { StatefulPipelineStack } from '../lib/pipeline/orcabus-stateful-pipeline-stack';
 
 const AWS_TOOLCHAIN_ACCOUNT = '383856791668'; // Bastion
 const AWS_TOOLCHAIN_REGION = 'ap-southeast-2';
 
 const app = new cdk.App();
 
-new PipelineStack(app, `OrcaBusPipeline`, {
+new StatefulPipelineStack(app, `OrcaBusStatefulPipeline`, {
   env: {
     account: AWS_TOOLCHAIN_ACCOUNT,
     region: AWS_TOOLCHAIN_REGION,
   },
   tags: {
-    'umccr-org:Stack': 'OrcaBusPipelineApp',
+    'umccr-org:Stack': 'OrcaBusStatefulPipelineApp',
     'umccr-org:Product': 'OrcaBus',
   },
 });

bin/stateless-pipeline.ts

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+import 'source-map-support/register';
+
+import * as cdk from 'aws-cdk-lib';
+import { StatelessPipelineStack } from '../lib/pipeline/orcabus-stateless-pipeline-stack';
+
+const AWS_TOOLCHAIN_ACCOUNT = '383856791668'; // Bastion
+const AWS_TOOLCHAIN_REGION = 'ap-southeast-2';
+
+const app = new cdk.App();
+
+new StatelessPipelineStack(app, `OrcaBusStatelessPipeline`, {
+  env: {
+    account: AWS_TOOLCHAIN_ACCOUNT,
+    region: AWS_TOOLCHAIN_REGION,
+  },
+  tags: {
+    'umccr-org:Stack': 'OrcaBusStatelessPipeline',
+    'umccr-org:Product': 'OrcaBus',
+  },
+});

cdk.json

@@ -1,5 +1,5 @@
 {
-  "app": "npx ts-node --prefer-ts-exts bin/pipeline.ts",
+  "app": "yarn run ts-node --prefer-ts-exts bin/orcabus.ts",
   "watch": {
     "include": [
       "**"

config/constants.ts

@@ -1,7 +1,7 @@
 import { OrcaBusStatefulConfig } from '../lib/workload/orcabus-stateful-stack';
 import { AuroraPostgresEngineVersion } from 'aws-cdk-lib/aws-rds';
 import { OrcaBusStatelessConfig } from '../lib/workload/orcabus-stateless-stack';
-import { Duration, aws_lambda } from 'aws-cdk-lib';
+import { Duration, aws_lambda, RemovalPolicy } from 'aws-cdk-lib';
 
 const regName = 'OrcaBusSchemaRegistry';
 const eventBusName = 'OrcaBusMain';
@@ -24,7 +24,7 @@ const orcaBusStatefulConfig = {
     defaultDatabaseName: 'orcabus',
     version: AuroraPostgresEngineVersion.VER_15_4,
     parameterGroupName: 'default.aurora-postgresql15',
-    username: 'admin',
+    username: 'postgres',
     dbPort: 5432,
     masterSecretName: rdsMasterSecretName,
     monitoring: {
@@ -94,6 +94,7 @@ export const getEnvironmentConfig = (
               maxACU: 1,
               enhancedMonitoringInterval: Duration.seconds(60),
               enablePerformanceInsights: true,
+              removalPolicy: RemovalPolicy.DESTROY,
             },
             securityGroupProps: {
               ...orcaBusStatefulConfig.securityGroupProps,
@@ -123,6 +124,7 @@ export const getEnvironmentConfig = (
               maxACU: 1,
               enhancedMonitoringInterval: Duration.seconds(60),
               enablePerformanceInsights: true,
+              removalPolicy: RemovalPolicy.DESTROY,
             },
             securityGroupProps: {
               ...orcaBusStatefulConfig.securityGroupProps,
@@ -150,6 +152,7 @@ export const getEnvironmentConfig = (
               numberOfInstance: 1,
               minACU: 0.5,
               maxACU: 1,
+              removalPolicy: RemovalPolicy.RETAIN,
             },
             securityGroupProps: {
               ...orcaBusStatefulConfig.securityGroupProps,

jest.config.js

@@ -5,4 +5,14 @@ module.exports = {
   transform: {
     '^.+\\.tsx?$': 'ts-jest',
   },
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        outputDirectory: 'target/report',
+        outputName: 'infrastructureTest.xml',
+      },
+    ],
+  ],
 };

lib/pipeline/orcabus-pipeline-stack.ts → lib/pipeline/orcabus-stateful-pipeline-stack.ts

@@ -3,12 +3,11 @@ import * as cdk from 'aws-cdk-lib';
 import * as ssm from 'aws-cdk-lib/aws-ssm';
 import * as pipelines from 'aws-cdk-lib/pipelines';
 import * as codebuild from 'aws-cdk-lib/aws-codebuild';
-import { OrcaBusStatelessConfig, OrcaBusStatelessStack } from '../workload/orcabus-stateless-stack';
+import * as iam from 'aws-cdk-lib/aws-iam';
 import { OrcaBusStatefulConfig, OrcaBusStatefulStack } from '../workload/orcabus-stateful-stack';
 import { getEnvironmentConfig } from '../../config/constants';
-import { PolicyStatement } from 'aws-cdk-lib/aws-iam';
 
-export class PipelineStack extends cdk.Stack {
+export class StatefulPipelineStack extends cdk.Stack {
   constructor(scope: Construct, id: string, props: cdk.StackProps) {
     super(scope, id, props);
 
@@ -18,34 +17,61 @@ export class PipelineStack extends cdk.Stack {
       connectionArn: codeStarArn,
     });
 
-    const synthAction = new pipelines.CodeBuildStep('Synth', {
+    const unitTest = new pipelines.CodeBuildStep('UnitTest', {
+      commands: ['yarn install --frozen-lockfile', 'make test-stateful'],
       input: sourceFile,
-      commands: ['yarn install --frozen-lockfile', 'yarn cdk synth -v'],
+      primaryOutputDirectory: '.',
+      buildEnvironment: {
+        privileged: true,
+        computeType: codebuild.ComputeType.LARGE,
+        buildImage: codebuild.LinuxArmBuildImage.AMAZON_LINUX_2_STANDARD_3_0,
+        environmentVariables: {
+          NODE_OPTIONS: {
+            value: '--max-old-space-size=8192',
+          },
+        },
+      },
+      partialBuildSpec: codebuild.BuildSpec.fromObject({
+        'orcabus-infrastructureStatefulReports': {
+          infrastructureStatefulReports: {
+            files: ['target/report/*.xml'],
+            'file-format': 'JUNITXML',
+          },
+        },
+        version: '0.2',
+      }),
+    });
+
+    const synthAction = new pipelines.CodeBuildStep('Synth', {
+      commands: ['yarn install --frozen-lockfile', 'yarn run cdk-stateful-pipeline synth'],
+      input: unitTest,
       primaryOutputDirectory: 'cdk.out',
       rolePolicyStatements: [
-        new PolicyStatement({
+        new iam.PolicyStatement({
+          effect: iam.Effect.ALLOW,
           actions: ['sts:AssumeRole'],
           resources: ['*'],
-          conditions: {
-            StringEquals: {
-              'iam:ResourceTag/aws-cdk:bootstrap-role': 'lookup',
-            },
-          },
         }),
       ],
     });
-    synthAction.addStepDependency(new OrcaBusTestStep('OrcaBusUnitTest', { source: sourceFile }));
 
     const pipeline = new pipelines.CodePipeline(this, 'Pipeline', {
       synth: synthAction,
       selfMutation: true,
       crossAccountKeys: true,
+      dockerEnabledForSynth: true,
+      dockerEnabledForSelfMutation: true,
       codeBuildDefaults: {
         buildEnvironment: {
-          buildImage: codebuild.LinuxBuildImage.STANDARD_6_0,
+          computeType: codebuild.ComputeType.LARGE,
+          buildImage: codebuild.LinuxArmBuildImage.AMAZON_LINUX_2_STANDARD_3_0,
+          environmentVariables: {
+            NODE_OPTIONS: {
+              value: '--max-old-space-size=8192',
+            },
+          },
         },
       },
-      dockerEnabledForSelfMutation: true,
     });
 
     /**
@@ -54,7 +80,7 @@ export class PipelineStack extends cdk.Stack {
     const betaConfig = getEnvironmentConfig('beta');
     if (!betaConfig) throw new Error(`No 'Beta' account configuration`);
     pipeline.addStage(
-      new OrcaBusDeploymentStage(this, 'BetaDeployment', betaConfig.stackProps, {
+      new OrcaBusStatefulDeploymentStage(this, 'BetaDeployment', betaConfig.stackProps, {
         account: betaConfig.accountId,
       })
     );
@@ -65,7 +91,7 @@ export class PipelineStack extends cdk.Stack {
     const gammaConfig = getEnvironmentConfig('gamma');
     if (!gammaConfig) throw new Error(`No 'Gamma' account configuration`);
     pipeline.addStage(
-      new OrcaBusDeploymentStage(this, 'GammaDeployment', gammaConfig.stackProps, {
+      new OrcaBusStatefulDeploymentStage(this, 'GammaDeployment', gammaConfig.stackProps, {
         account: gammaConfig.accountId,
       }),
       { pre: [new pipelines.ManualApprovalStep('PromoteToGamma')] }
@@ -74,65 +100,28 @@ export class PipelineStack extends cdk.Stack {
     /**
      * Deployment to Prod account (DISABLED)
      */
-    // const prodConfig = getEnvironmentConfig('prod');
-    // if (!prodConfig) throw new Error(`No 'Prod' account configuration`);
-    // pipeline.addStage(
-    //   new OrcaBusDeploymentStage(this, 'prodDeployment', prodConfig.stackProps, {
-    //     account: gammaConfig?.accountId,
-    //   }),
-    //   { pre: [new pipelines.ManualApprovalStep('PromoteToProd')] }
-    // );
+    const prodConfig = getEnvironmentConfig('prod');
+    if (!prodConfig) throw new Error(`No 'Prod' account configuration`);
+    pipeline.addStage(
+      new OrcaBusStatefulDeploymentStage(this, 'prodDeployment', prodConfig.stackProps, {
+        account: gammaConfig?.accountId,
+      }),
+      { pre: [new pipelines.ManualApprovalStep('PromoteToProd')] }
+    );
   }
 }
 
-class OrcaBusDeploymentStage extends cdk.Stage {
+class OrcaBusStatefulDeploymentStage extends cdk.Stage {
   constructor(
     scope: Construct,
     environmentName: string,
     stackProps: {
       orcaBusStatefulConfig: OrcaBusStatefulConfig;
-      orcaBusStatelessConfig: OrcaBusStatelessConfig;
     },
     env?: cdk.Environment
   ) {
     super(scope, environmentName, { env: { account: env?.account, region: 'ap-southeast-2' } });
 
     new OrcaBusStatefulStack(this, 'OrcaBusStatefulStack', stackProps.orcaBusStatefulConfig);
-    new OrcaBusStatelessStack(this, 'OrcaBusStatelessStack', stackProps.orcaBusStatelessConfig);
-  }
-}
-
-interface OrcaBusTestStepProps {
-  source: pipelines.CodePipelineSource;
-}
-class OrcaBusTestStep extends pipelines.CodeBuildStep {
-  constructor(id: string, props: OrcaBusTestStepProps) {
-    const stepProps: pipelines.CodeBuildStepProps = {
-      input: props.source,
-      commands: [],
-      partialBuildSpec: codebuild.BuildSpec.fromObject({
-        env: {
-          shell: 'bash',
-        },
-        phases: {
-          install: {
-            commands: [
-              'wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py310_23.3.1-0-Linux-x86_64.sh',
-              'bash Miniconda3-py310_23.3.1-0-Linux-x86_64.sh -b',
-              'export PATH=/root/miniconda3/bin:$PATH',
-              'conda init bash',
-              'source activate',
-              'conda create -q -n orcabus python=3.10',
-              'conda run -n orcabus yarn install',
-            ],
-          },
-          build: {
-            commands: ['conda run -n orcabus make test'],
-          },
-        },
-        version: '0.2',
-      }),
-    };
-    super(id, stepProps);
   }
 }