Merge pull request #14 from prudhvigodithi/main

OpenSearch Metrics with OSD: Add tests and Cluster Acess for Jenkins

.github/workflows/ci-test.yml

@@ -0,0 +1,25 @@
+name: Build and Test the Code
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - "*"
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Java 17
+        uses: actions/setup-java@v2
+        with:
+          java-version: '17'
+
+      - name: Run build and test
+        run: |
+          ./gradlew clean build

.gitignore

@@ -18,6 +18,7 @@
 *.zip
 *.tar.gz
 *.rar
+!gradle-wrapper.jar
 
 # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
 hs_err_pid*

build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id 'java'
-    //id 'com.github.johnrengelman.shadow' version '7.1.2'
+    id 'jacoco'
 }
 
 group 'org.opensearch.opensearchmetrics'
@@ -10,29 +10,24 @@ repositories {
     mavenCentral()
 }
 
-//shadowJar {
-//    archiveClassifier = 'opensearch'
-//}
+jacoco {
+    toolVersion = "0.8.11"
+}
 
 dependencies {
     compileOnly 'org.projectlombok:lombok:1.18.30'
     annotationProcessor 'org.projectlombok:lombok:1.18.30'
     implementation 'com.google.guava:guava:32.0.1-jre'
 
-    implementation 'com.google.dagger:dagger:2.48'
+    implementation 'com.google.dagger:dagger:2.51'
     annotationProcessor 'com.google.dagger:dagger-compiler:2.48'
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.3'
-    //implementation 'org.opensearch.client:opensearch-java:2.8.1'
     implementation 'org.opensearch.client:opensearch-rest-high-level-client:2.11.0'
     implementation 'org.apache.httpcomponents.client5:httpclient5:5.2.1'
-    //implementation 'com.amazonaws:aws-java-sdk-sts:1.12.599'
     implementation 'software.amazon.awssdk:sts:2.21.33'
-    // implementation 'com.amazonaws:aws-java-sdk-core:1.12.599'
     implementation 'io.github.acm19:aws-request-signing-apache-interceptor:2.3.1'
 
     implementation 'com.amazonaws:aws-lambda-java-core:1.2.3'
-    // implementation 'com.amazonaws:aws-java-sdk-sns:1.12.418'
-    // implementation 'com.amazonaws:aws-lambda-java-events:3.11.0'
 
     implementation 'com.google.code.gson:gson:2.10.1'
 
@@ -56,6 +51,15 @@ build.dependsOn buildZip
 
 test {
     useJUnitPlatform()
+    finalizedBy jacocoTestReport
+}
+
+jacocoTestReport {
+    reports {
+        xml.enabled true
+        html.enabled true
+        html.outputLocation = layout.buildDirectory.dir('jacocoHtml')
+    }
 }
 
 sourceSets {
@@ -65,3 +69,5 @@ sourceSets {
         }
     }
 }
+
+

infrastructure/lib/constructs/opensearchNginxProxyCognito.ts

@@ -1,4 +1,10 @@
-import { BlockDeviceVolume, CfnLaunchConfiguration, HealthCheck, UpdatePolicy, AutoScalingGroup } from 'aws-cdk-lib/aws-autoscaling';
+import {
+    BlockDeviceVolume,
+    HealthCheck,
+    UpdatePolicy,
+    AutoScalingGroup,
+    CfnLaunchConfiguration
+} from 'aws-cdk-lib/aws-autoscaling';
 import {
     InstanceClass,
     InstanceSize,
@@ -12,21 +18,28 @@ import {
     AmazonLinuxImage
 } from 'aws-cdk-lib/aws-ec2';
 import { Effect, ManagedPolicy, PolicyStatement, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam';
-import {Aspects, Duration, Tag, Tags} from 'aws-cdk-lib';
+import {Aspects, CfnOutput, Duration, Tag, Tags} from 'aws-cdk-lib';
 import { Construct } from 'constructs';
-import {ListenerCertificate, NetworkLoadBalancer, Protocol} from "aws-cdk-lib/aws-elasticloadbalancingv2";
+import {
+    ApplicationLoadBalancer, ApplicationProtocol,
+    ListenerCertificate,
+} from "aws-cdk-lib/aws-elasticloadbalancingv2";
 import Project from "../enums/project";
+import {ARecord, RecordTarget} from "aws-cdk-lib/aws-route53";
+import {LoadBalancerTarget} from "aws-cdk-lib/aws-route53-targets";
+import {OpenSearchHealthRoute53} from "../stacks/route53";
 
 
 export interface NginxProps {
     readonly vpc: Vpc;
     readonly securityGroup: SecurityGroup;
     readonly opensearchDashboardUrlProps: opensearchDashboardUrlProps;
-    readonly nlbProps?: nlbProps
+    readonly albProps?: albProps
     readonly region: string;
 }
 
-export interface nlbProps {
+export interface albProps {
+    hostedZone: OpenSearchHealthRoute53;
     certificateArn: string,
 }
 
@@ -52,14 +65,14 @@ export class OpenSearchMetricsNginxCognito extends Construct {
             machineImage: new AmazonLinuxImage({
                 generation: AmazonLinuxGeneration.AMAZON_LINUX_2,
             }),
-            // Temp added private subnet
-            // associatePublicIpAddress: true,
+            // Temp added public subnet and IP, until backed up by ALB
             associatePublicIpAddress: true,
             allowAllOutbound: true,
             desiredCapacity: 1,
             minCapacity: 1,
             vpc: props.vpc,
             vpcSubnets: {
+                // Temp added public subnet and IP, until backed up by ALB
                // subnetType: SubnetType.PUBLIC,
                 subnetType: SubnetType.PUBLIC
             },
@@ -68,48 +81,49 @@ export class OpenSearchMetricsNginxCognito extends Construct {
             // and then destroy the old one in order to maintain availability
             updatePolicy: UpdatePolicy.replacingUpdate()
         });
-        this.asg.addSecurityGroup(props.securityGroup);
         Tags.of(this.asg).add("Name", "OpenSearchMetricsCognito")
 
-        // Allow traffic from the VPC
-        this.asg.connections.allowFrom(Peer.ipv4(props.vpc.vpcCidrBlock), Port.allTcp(), 'Local VPC Access');
-
-        if (props.nlbProps) {
-            const lb = new NetworkLoadBalancer(this, `OpenSearchMetricsCognito-NginxProxyNlb`, {
+        if (props.albProps) {
+            const openSearchCognitoApplicationLoadBalancer = new ApplicationLoadBalancer(this, `OpenSearchMetricsCognito-NginxProxyAlb`, {
                 loadBalancerName: "OpenSearchMetricsCognito",
                 vpc: vpc,
                 internetFacing: true
             });
 
-            const listenerCertificate = ListenerCertificate.fromArn(props.nlbProps.certificateArn);
+            const listenerCertificate = ListenerCertificate.fromArn(props.albProps.certificateArn);
 
-            const listener = lb.addListener(`OpenSearchMetricsCognito-NginxProxyNlbListener`, {
+            const listener = openSearchCognitoApplicationLoadBalancer.addListener(`OpenSearchMetricsCognito-NginxProxyAlbListener`, {
                 port: 443,
-                protocol: Protocol.TLS,
+                protocol: ApplicationProtocol.HTTPS,
                 certificates: [listenerCertificate]
             });
 
-            listener.addTargets(`OpenSearchMetricsCognito-NginxProxyNlbTarget`, {
+            listener.addTargets(`OpenSearchMetricsCognito-NginxProxyAlbTarget`, {
                 port: 443,
+                protocol: ApplicationProtocol.HTTPS,
                 targets: [this.asg]
             });
+
+
+            const aRecord = new ARecord(this, "OpenSearchMetricsCognito-DNS", {
+                zone: props.albProps.hostedZone.zone,
+                recordName: Project.METRICS_HOSTED_ZONE,
+                target: RecordTarget.fromAlias(new LoadBalancerTarget(openSearchCognitoApplicationLoadBalancer)),
+            });
         }
 
-        // Enforces IMDSv2
         const launchConfiguration = this.asg.node.findChild('LaunchConfig') as CfnLaunchConfiguration;
         launchConfiguration.metadataOptions = {
             httpPutResponseHopLimit: 2,
             httpEndpoint: "enabled",
             httpTokens: "required"
         };
 
-        this.asg.addSecurityGroup(securityGroup);
-        // Allow traffic from the VPC
-        this.asg.connections.allowFrom(
-            Peer.ipv4(vpc.vpcCidrBlock),
-            Port.allTcp(),
-            "Local VPC Access"
-        );
+        // To ensure the Cfn outputs are not deleted
+        new CfnOutput(this, 'VpcCidr', {
+            value: vpc.vpcCidrBlock,
+            description: 'VPC CIDR',
+        });
 
         this.asg.connections.allowFrom(
             Peer.prefixList(Project.RESTRICTED_PREFIX),
@@ -132,7 +146,6 @@ export class OpenSearchMetricsNginxCognito extends Construct {
             assumedBy: new ServicePrincipal('ec2.amazonaws.com'),
             roleName: "OpenSearchCognitoUserAccess",
         });
-        // SSM integration - https://aws.amazon.com/systems-manager/
         role.addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'));
         return role;
     }

infrastructure/lib/enums/project.ts

@@ -1,5 +1,7 @@
 enum Project{
     AWS_ACCOUNT = '',
+    JENKINS_MASTER_ROLE = '',
+    JENKINS_AGENT_ROLE = '',
     REGION = '',
     METRICS_HOSTED_ZONE = 'metrics.opensearch.org',
     // Temp until the project is public

infrastructure/lib/infrastructure-stack.ts

@@ -1,11 +1,12 @@
 import {App, CfnOutput, Stack, StackProps} from 'aws-cdk-lib';
 import { Construct } from 'constructs';
 import { VpcStack } from "./stacks/vpc";
-import { OpenSearchDomainStack } from "./stacks/opensearch";
+import {jenkinsAccess, OpenSearchDomainStack} from "./stacks/opensearch";
 import Project from './enums/project';
 import {OpenSearchHealthRoute53} from "./stacks/route53";
 import {OpenSearchMetricsWorkflowStack} from "./stacks/metricsWorkflow";
 import {OpenSearchMetricsNginxReadonly} from "./stacks/opensearchNginxProxyReadonly";
+import {ArnPrincipal, IPrincipal} from "aws-cdk-lib/aws-iam";
 
 // import * as sqs from 'aws-cdk-lib/aws-sqs';
 export class InfrastructureStack extends Stack {
@@ -17,12 +18,18 @@ export class InfrastructureStack extends Stack {
     const vpcStack = new VpcStack(app, "OpenSearchHealth-VPC", {});
 
 
-    // Create OpenSearch Domain, roles, permissions, cognito setup
+    // Create OpenSearch Domain, roles, permissions, cognito setup, cross account OpenSearch access for jenkins
     const openSearchDomainStack = new OpenSearchDomainStack(app, "OpenSearchHealth-OpenSearch", {
       region: Project.REGION,
       account: Project.AWS_ACCOUNT,
       vpcStack: vpcStack,
       enableNginxCognito: true,
+      jenkinsAccess: {
+        jenkinsAccountRoles:  [
+          new ArnPrincipal(Project.JENKINS_MASTER_ROLE),
+          new ArnPrincipal(Project.JENKINS_AGENT_ROLE)
+        ]
+      }
     });
 
 
@@ -48,7 +55,7 @@ export class InfrastructureStack extends Stack {
         opensearchDashboardVpcUrl: openSearchDomainStack.domain.domainEndpoint,
         openSearchDomainName: openSearchDomainStack.domain.domainName
       },
-      nlbProps: {
+      albProps: {
         hostedZone: metricsHostedZone,
         certificateArn: metricsHostedZone.certificateArn,
       },