Skip to content

Commit

Permalink
Added functionality to store github events in s3 bucket
Browse files Browse the repository at this point in the history
Signed-off-by: Brandon Shien <bshien@amazon.com>
  • Loading branch information
bshien committed Oct 9, 2024
1 parent a6eb945 commit f5a4c7f
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 8 deletions.
1 change: 1 addition & 0 deletions DEVELOPER_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ So you want to contribute code to this project? Excellent! We're glad you're her
- `cdk deploy OpenSearchMetrics-GitHubAutomationApp-Secret`: Creates the GitHub app secret which will be used during the GitHub app runtime.
- `cdk deploy OpenSearchMetrics-GitHubWorkflowMonitor-Alarms`: Creates the Alarms to Monitor the Critical GitHub CI workflows by the GitHub Automation App.
- `cdk deploy OpenSearchMetrics-GitHubAutomationApp`: Create the resources which launches the [GitHub Automation App](https://github.com/opensearch-project/automation-app). Listens to GitHub events and index the data to Metrics cluster.
- `cdk deploy OpenSearchMetrics-GitHubAutomationAppEvents-S3`: Creates the S3 Bucket for the [GitHub Automation App](https://github.com/opensearch-project/automation-app) to store OpenSearch Project GitHub Events.

### Forking and Cloning

Expand Down
9 changes: 7 additions & 2 deletions infrastructure/lib/infrastructure-stack.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { OpenSearchMetricsNginxCognito } from "./constructs/opensearchNginxProxy
import { OpenSearchMetricsMonitoringStack } from "./stacks/monitoringDashboard";
import { OpenSearchMetricsSecretsStack } from "./stacks/secrets";
import { GitHubAutomationApp } from "./stacks/gitHubAutomationApp";
import { OpenSearchS3 } from "./stacks/s3";
import { GitHubWorkflowMonitorAlarms } from "./stacks/gitHubWorkflowMonitorAlarms";

export class InfrastructureStack extends Stack {
Expand Down Expand Up @@ -47,15 +48,19 @@ export class InfrastructureStack extends Stack {
],
});

// Create S3 bucket for the GitHub Events
const openSearchEventsS3Bucket = new OpenSearchS3(app, "OpenSearchMetrics-GitHubAutomationAppEvents-S3");

// Create resources to launch the GitHub Automation App
const gitHubAutomationApp = new GitHubAutomationApp(app, "OpenSearchMetrics-GitHubAutomationApp", {
vpc: vpcStack.vpc,
region: Project.REGION,
account: Project.AWS_ACCOUNT,
ami: Project.EC2_AMI_SSM.toString(),
secret: openSearchMetricsGitHubAutomationAppSecretStack.secret,
workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn
})
workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn,
githubEventsBucketArn: openSearchEventsS3Bucket.bucket.bucketArn
});


// Create OpenSearch Domain, roles, permissions, cognito setup, cross account OpenSearch access for jenkins
Expand Down
19 changes: 15 additions & 4 deletions infrastructure/lib/stacks/gitHubAutomationApp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ export interface GitHubAppProps {
readonly ami?: string
readonly secret: Secret;
readonly workflowAlarmsArn: string[];
readonly githubEventsBucketArn: string;
}


Expand All @@ -53,11 +54,11 @@ export class GitHubAutomationApp extends Stack {
constructor(scope: Construct, id: string, props: GitHubAppProps) {
super(scope, id);

const instanceRole = this.createInstanceRole(props.secret.secretArn, props.account, props.workflowAlarmsArn);
const instanceRole = this.createInstanceRole(props.secret.secretArn, props.account, props.workflowAlarmsArn, props.githubEventsBucketArn);
this.githubAppRole = instanceRole;

this.asg = new AutoScalingGroup(this, 'OpenSearchMetrics-GitHubAutomationApp-Asg', {
instanceType: InstanceType.of(InstanceClass.M5, InstanceSize.LARGE),
instanceType: InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE),
blockDevices: [{deviceName: '/dev/xvda', volume: BlockDeviceVolume.ebs(20)}],
healthCheck: HealthCheck.ec2({grace: Duration.seconds(90)}),
machineImage: props && props.ami ?
Expand Down Expand Up @@ -92,7 +93,7 @@ export class GitHubAutomationApp extends Stack {
this.asg.addUserData(...this.getUserData(props.secret.secretName));
}

private createInstanceRole(secretArn: string, account: string, alarmsArn: string[]): Role {
private createInstanceRole(secretArn: string, account: string, alarmsArn: string[], githubEventsBucketArn: string): Role {
const role = new Role(this, "OpenSearchMetrics-GitHubAutomationApp-Role", {
assumedBy: new CompositePrincipal(
new ServicePrincipal('ec2.amazonaws.com'),
Expand Down Expand Up @@ -137,6 +138,15 @@ export class GitHubAutomationApp extends Stack {

}),
);
role.addToPolicy(
new PolicyStatement({
effect: Effect.ALLOW,
actions: [
"s3:PutObject",
],
resources: [`${githubEventsBucketArn}/*`],
}),
);
return role;
}

Expand All @@ -154,7 +164,8 @@ export class GitHubAutomationApp extends Stack {
`aws secretsmanager get-secret-value --secret-id ${secretName} --query SecretString --output text >> automation-app/.env`,
'cd automation-app/docker',
'PORT=8080 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-merged-pulls-monitor.yml docker-compose -p github-merged-pulls-monitor up -d',
'PORT=8081 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-workflow-runs-monitor.yml docker-compose -p github-workflow-runs-monitor up -d'
'PORT=8081 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-workflow-runs-monitor.yml docker-compose -p github-workflow-runs-monitor up -d',
'PORT=8082 RESOURCE_CONFIG=configs/resources/opensearch-project-resource.yml OPERATION_CONFIG=configs/operations/github-events-to-s3.yml docker-compose -p github-events-to-s3 up -d',
];
}
}
29 changes: 29 additions & 0 deletions infrastructure/lib/stacks/s3.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/**
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

import {Stack, RemovalPolicy, StackProps} from "aws-cdk-lib";
import { Bucket, BlockPublicAccess, BucketEncryption } from 'aws-cdk-lib/aws-s3';
import { Construct } from "constructs";

export class OpenSearchS3 extends Stack {

public readonly bucket: Bucket;

constructor(scope: Construct, id: string, props?: StackProps) {
super(scope, id);

this.bucket = new Bucket(this, 'OpenSearchS3Bucket', {
publicReadAccess: false,
blockPublicAccess: BlockPublicAccess.BLOCK_ALL,
encryption: BucketEncryption.S3_MANAGED,
enforceSSL: true,
versioned: true,
removalPolicy: RemovalPolicy.RETAIN,
});
}
}
1 change: 0 additions & 1 deletion infrastructure/lib/stacks/vpc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import { Stack, StackProps } from "aws-cdk-lib";
import { Peer, Port, SecurityGroup, SelectedSubnets, SubnetType, Vpc } from 'aws-cdk-lib/aws-ec2';
import { Construct } from "constructs";


export class VpcStack extends Stack {
public readonly vpc: Vpc;
public readonly securityGroup: SecurityGroup
Expand Down
27 changes: 26 additions & 1 deletion infrastructure/test/gitHubAutomationApp-stack.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { VpcStack } from "../lib/stacks/vpc";
import { GitHubAutomationApp } from "../lib/stacks/gitHubAutomationApp";
import { OpenSearchMetricsSecretsStack } from "../lib/stacks/secrets";
import {GitHubWorkflowMonitorAlarms} from "../lib/stacks/gitHubWorkflowMonitorAlarms";
import {OpenSearchS3} from "../lib/stacks/s3";


test('OpenSearch GitHub App Stack test ', () => {
Expand All @@ -21,6 +22,7 @@ test('OpenSearch GitHub App Stack test ', () => {
const openSearchMetricsGitHubAppSecretStack = new OpenSearchMetricsSecretsStack(app, "Test-OpenSearchMetrics-GitHubAutomationApp-Secret", {
secretName: 'test-github-app-creds'
});
const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3");

const gitHubWorkflowMonitorAlarms = new GitHubWorkflowMonitorAlarms(app, "Test-OpenSearchMetrics-GitHubWorkflowMonitor-Alarms", {
namespace: 'GitHubActions',
Expand All @@ -36,7 +38,8 @@ test('OpenSearch GitHub App Stack test ', () => {
account: Project.AWS_ACCOUNT,
ami: Project.EC2_AMI_SSM.toString(),
secret: openSearchMetricsGitHubAppSecretStack.secret,
workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn
workflowAlarmsArn: gitHubWorkflowMonitorAlarms.workflowAlarmsArn,
githubEventsBucketArn: s3Stack.bucket.bucketArn
});

const template = Template.fromStack(gitHubApp);
Expand Down Expand Up @@ -90,4 +93,26 @@ test('OpenSearch GitHub App Stack test ', () => {
])
}
});

template.hasResourceProperties('AWS::IAM::Policy', {
PolicyDocument: {
Statement: Match.arrayWith([
Match.objectLike({
Action: "s3:PutObject",
Effect: "Allow",
Resource: {
"Fn::Join": [
"",
[
{
"Fn::ImportValue": Match.stringLikeRegexp('Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3:ExportsOutputFnGetAttOpenSearchS3Bucket.*')
},
"/*"
]
]
}
})
])
}
});
});
84 changes: 84 additions & 0 deletions infrastructure/test/s3-stack.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/**
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

import { App } from "aws-cdk-lib";
import { Template } from "aws-cdk-lib/assertions";
import {OpenSearchS3} from "../lib/stacks/s3";

test('S3 Stack Test', () => {
const app = new App();
const s3Stack = new OpenSearchS3(app, "Test-OpenSearchMetrics-GitHubAutomationAppEvents-S3");
const s3StackTemplate = Template.fromStack(s3Stack);
s3StackTemplate.resourceCountIs('AWS::S3::Bucket', 1);
s3StackTemplate.resourceCountIs('AWS::S3::BucketPolicy', 1);
s3StackTemplate.hasResourceProperties('AWS::S3::Bucket', {
"BucketEncryption": {
"ServerSideEncryptionConfiguration": [
{
"ServerSideEncryptionByDefault": {
"SSEAlgorithm": "AES256"
}
}
]
},
"PublicAccessBlockConfiguration": {
"BlockPublicAcls": true,
"BlockPublicPolicy": true,
"IgnorePublicAcls": true,
"RestrictPublicBuckets": true
},
"VersioningConfiguration": {
"Status": "Enabled"
}
});
s3StackTemplate.hasResourceProperties('AWS::S3::BucketPolicy', {
"Bucket": {
"Ref": "OpenSearchS3Bucket2ED683CC"
},
"PolicyDocument": {
"Statement": [
{
"Action": "s3:*",
"Condition": {
"Bool": {
"aws:SecureTransport": "false"
}
},
"Effect": "Deny",
"Principal": {
"AWS": "*"
},
"Resource": [
{
"Fn::GetAtt": [
"OpenSearchS3Bucket2ED683CC",
"Arn"
]
},
{
"Fn::Join": [
"",
[
{
"Fn::GetAtt": [
"OpenSearchS3Bucket2ED683CC",
"Arn"
]
},
"/*"
]
]
}
]
}
],
"Version": "2012-10-17"
}
});

});

0 comments on commit f5a4c7f

Please sign in to comment.