From 4f4035153b5fac6f14bfb89c011be3d3341a4315 Mon Sep 17 00:00:00 2001 From: Christopher Hein Date: Tue, 21 Aug 2018 03:46:35 -0700 Subject: [PATCH] Adding S3 Bucket Service Using ExternalName **Why:** * Allows local traffic to be configured via operator internally * Updates sqs files to sqsqueues **This change addresses the need by:** * closes #47 Signed-off-by: Christopher Hein --- cloudformation/dynamodb.yaml | 93 ++++++++++++++++++ cloudformation/s3bucket.yaml | 141 +++++++++++++++++++++++++++ cloudformation/snstopic.yaml | 31 ++++++ cloudformation/sqsqueue.yaml | 135 +++++++++++++++++++++++++ examples/s3bucket.yaml | 4 +- examples/{sqs.yaml => sqsqueue.yaml} | 0 models/s3bucket.yaml | 22 +++-- pkg/helpers/service.go | 1 + 8 files changed, 418 insertions(+), 9 deletions(-) create mode 100644 cloudformation/dynamodb.yaml create mode 100644 cloudformation/s3bucket.yaml create mode 100644 cloudformation/snstopic.yaml create mode 100644 cloudformation/sqsqueue.yaml rename examples/{sqs.yaml => sqsqueue.yaml} (100%) diff --git a/cloudformation/dynamodb.yaml b/cloudformation/dynamodb.yaml new file mode 100644 index 000000000..553eec6d8 --- /dev/null +++ b/cloudformation/dynamodb.yaml @@ -0,0 +1,93 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: 'AWS Operator - Amazon DynamoDB' +Parameters: + Namespace: + Description: >- + This is the namespace for the Kubernetes object. + Type: String + ResourceVersion: + Type: String + Description: >- + This is the resource version for the Kubernetes object. + ResourceName: + Description: >- + This is the resource name for the Kubernetes object + Type: String + ClusterName: + Description: >- + This is the cluster name for the operator + Type: String + TableName: + Description: >- + Must contain only lowercase letters, numbers and hyphens. + Type: String + HashAttributeName: + Type: String + Description: Name of the Hash key + HashAttributeType: + Type: String + AllowedValues: + - S + - N + - B + Default: "S" + Description: AttributeType for Hash key + RangeAttributeName: + Type: String + Description: Name of the Range key + RangeAttributeType: + Type: String + AllowedValues: + - S + - N + - B + Default: "S" + Description: AttributeType for the Range key + ReadCapacityUnits: + Type: String + Description: Read ReadCapacity Units + Default: "5" + WriteCapacityUnits: + Type: String + Description: Write Capacity Units + Default: "5" +Resources: + DynamoDBTable: + Type: "AWS::DynamoDB::Table" + Properties: + TableName: !Ref TableName + KeySchema: + - + AttributeName: !Ref HashAttributeName + KeyType: "HASH" + - + AttributeName: !Ref RangeAttributeName + KeyType: "RANGE" + AttributeDefinitions: + - + AttributeName: !Ref HashAttributeName + AttributeType: "S" + - + AttributeName: !Ref RangeAttributeName + AttributeType: "S" + ProvisionedThroughput: + ReadCapacityUnits: !Ref ReadCapacityUnits + WriteCapacityUnits: !Ref WriteCapacityUnits + Tags: + - Key: Namespace + Value: !Ref Namespace + - Key: ResourceVersion + Value: !Ref ResourceVersion + - Key: ResourceName + Value: !Ref ResourceName + - Key: ClusterName + Value: !Ref ClusterName + - Key: Heritage + Value: operator.aws +Outputs: + TableName: + Description: Name of the DynamoDB Table + Value: !Ref DynamoDBTable + TableArn: + Description: Arn of the DynamoDB Table + Value: !GetAtt DynamoDBTable.Arn diff --git a/cloudformation/s3bucket.yaml b/cloudformation/s3bucket.yaml new file mode 100644 index 000000000..6bfed3985 --- /dev/null +++ b/cloudformation/s3bucket.yaml @@ -0,0 +1,141 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: 'AWS Operator - Amazon S3 Bucket' +Parameters: + Namespace: + Description: >- + This is the namespace for the Kubernetes object. + Type: String + ResourceVersion: + Type: String + Description: >- + This is the resource version for the Kubernetes object. + ResourceName: + Description: >- + This is the resource name for the Kubernetes object + Type: String + ClusterName: + Description: >- + This is the cluster name for the operator + Type: String + BucketName: + Description: >- + Must contain only lowercase letters, numbers, periods (.), and hyphens + (-),Cannot end in numbers + Type: String + Default: apps3bucket + LoggingPrefix: + Description: >- + Must contain only lowercase letters, numbers, periods (.), and hyphens + (-),Cannot end in numbers + Type: String + Default: Archive + EnableLogging: + Description: enable or discable S3 logging + Type: String + AllowedValues: + - 'true' + - 'false' + Default: 'true' + EnableGlacierLifeCycle: + Description: enable archiving to Glacier Storage + Type: String + AllowedValues: + - 'true' + - 'false' + Default: 'false' + GlacierLifeCycleTransitionInDays: + Description: Define how many days objects should exist before being moved to Glacier + Type: String + Default: '0' + EnableVersioning: + Description: enable versioning + Type: String + AllowedValues: + - 'true' + - 'false' + Default: 'false' + LifeCyclePrefix: + Description: >- + Must contain only lowercase letters, numbers, periods (.), and hyphens + (-),Cannot end in numbers + Type: String + Default: Archive + BucketAccessControl: + Description: define if the bucket can be accessed from public or private locations + Type: String + AllowedValues: + - Private + - PublicRead + - PublicReadWrite + - AuthenticatedRead + - LogDeliveryWrite + - BucketOwnerRead + - BucketOwnerFullControl + - AwsExecRead + Default: "Private" +Mappings: {} +Conditions: + UseLogging: !Equals + - !Ref EnableLogging + - 'true' + UseGlacierLifeCycle: !Equals + - !Ref EnableGlacierLifeCycle + - 'true' + UseVersioning: !Equals + - !Ref EnableVersioning + - 'true' +Resources: + S3bucket: + Type: 'AWS::S3::Bucket' + Properties: + BucketName: !Ref BucketName + AccessControl: !Ref BucketAccessControl + LifecycleConfiguration: + Rules: + - Id: GlacierRule + Prefix: !Ref LifeCyclePrefix + Status: Enabled + ExpirationInDays: '365' + Transitions: + - TransitionInDays: !Ref GlacierLifeCycleTransitionInDays + StorageClass: Glacier + LoggingConfiguration: !If + - UseLogging + - DestinationBucketName: !Ref LoggingBucket + LogFilePrefix: !Ref LoggingPrefix + - !Ref 'AWS::NoValue' + Tags: + - Key: Namespace + Value: !Ref Namespace + - Key: ResourceVersion + Value: !Ref ResourceVersion + - Key: ResourceName + Value: !Ref ResourceName + - Key: ClusterName + Value: !Ref ClusterName + - Key: Heritage + Value: operator.aws + VersioningConfiguration: !If + - UseVersioning + - Status: Enabled + - !Ref 'AWS::NoValue' + DeletionPolicy: Retain + LoggingBucket: + Condition: UseLogging + Type: 'AWS::S3::Bucket' + DeletionPolicy: Retain + Properties: + AccessControl: LogDeliveryWrite + BucketName: !Join + - '' + - - !Ref BucketName + - logging +Outputs: + BucketName: + Value: !Ref S3bucket + Description: Name of the sample Amazon S3 bucket. + BucketArn: + Value: !GetAtt + - S3bucket + - Arn + Description: Name of the Amazon S3 bucket diff --git a/cloudformation/snstopic.yaml b/cloudformation/snstopic.yaml new file mode 100644 index 000000000..66a47ac54 --- /dev/null +++ b/cloudformation/snstopic.yaml @@ -0,0 +1,31 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: 'AWS Service Broker - Amazon SNS (qs-1nt0fs93c)' +Parameters: + Namespace: + Description: >- + This is the namespace for the Kubernetes object. + Type: String + ResourceVersion: + Type: String + Description: >- + This is the resource version for the Kubernetes object. + ResourceName: + Description: >- + This is the resource name for the Kubernetes object + Type: String + ClusterName: + Description: >- + This is the cluster name for the operator + Type: String + DisplayName: + Type: String + Description: >- + What should the SNS Topics name display as. +Resources: + SNSTopic: + Type: 'AWS::SNS::Topic' + +Outputs: + TopicName: + Value: !Ref SNSTopic + Description: Name of the topic diff --git a/cloudformation/sqsqueue.yaml b/cloudformation/sqsqueue.yaml new file mode 100644 index 000000000..678e10738 --- /dev/null +++ b/cloudformation/sqsqueue.yaml @@ -0,0 +1,135 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: "AWS Servicebroker - Amazon SQS (qs-1nt0fs93h)" +Parameters: + Namespace: + Type: String + Description: >- + This is the namespace for the Kubernetes object. + ResourceVersion: + Type: String + Description: >- + This is the resource version for the Kubernetes object. + ResourceName: + Type: String + Description: >- + This is the resource name for the Kubernetes object + ClusterName: + Type: String + Description: >- + This is the cluster name for the operator + ContentBasedDeduplication: + Type: String + Description: >- + Specifies whether to enable content-based deduplication, only applies to FIFO queues + AllowedValues: + - 'true' + - 'false' + Default: 'true' + DelaySeconds: + Type: Number + Description: >- + The Id of the AMI you wish to launch the instance from. + Default: '5' + MaximumMessageSize: + Type: Number + Description: >- + The limit of how many bytes that a message can contain before Amazon SQS + rejects it, 1024 bytes (1 KiB) to 262144 bytes (256 KiB) + Default: '262144' + MessageRetentionPeriod: + Type: Number + Description: >- + The number of seconds that Amazon SQS retains a message. You can specify + an integer value from 60 seconds (1 minute) to 1209600 seconds (14 days). + Default: '345600' + ReceiveMessageWaitTimeSeconds: + Type: Number + Description: >- + Specifies the duration, in seconds, that the ReceiveMessage action call + waits until a message is in the queue in order to include it in the + response, as opposed to returning an empty response if a message is not + yet available. 1 to 20 + Default: '0' + UsedeadletterQueue: + Type: String + Description: >- + A dead-letter queue is a queue that other (source) queues can target for + messages that can't be processed (consumed) successfully. You can set + aside and isolate these messages in the dead-letter queue to determine why + their processing doesn't succeed. + AllowedValues: + - 'true' + - 'false' + Default: 'false' + VisibilityTimeout: + Type: Number + Description: >- + This should be longer than the time it would take to process and delete a + message, this should not exceed 12 hours. + Default: '5' + FifoQueue: + Type: String + Description: >- + If true queue will be FIFO + AllowedValues: + - 'true' + - 'false' + Default: 'false' +Conditions: + CreateDeadLetterQueue: !Equals + - !Ref UsedeadletterQueue + - 'true' + IsFifo: !Equals + - !Ref FifoQueue + - 'true' +Resources: + SQSQueue: + Type: 'AWS::SQS::Queue' + Properties: + ContentBasedDeduplication: !If + - IsFifo + - !Ref ContentBasedDeduplication + - !Ref AWS::NoValue + FifoQueue: !If + - IsFifo + - 'true' + - !Ref AWS::NoValue + MaximumMessageSize: !Ref MaximumMessageSize + MessageRetentionPeriod: !Ref MessageRetentionPeriod + ReceiveMessageWaitTimeSeconds: !Ref ReceiveMessageWaitTimeSeconds + RedrivePolicy: !If + - CreateDeadLetterQueue + - deadLetterTargetArn: !GetAtt + - MyDeadLetterQueue + - Arn + maxReceiveCount: 5 + - !Ref 'AWS::NoValue' + VisibilityTimeout: !Ref VisibilityTimeout + MyDeadLetterQueue: + Condition: CreateDeadLetterQueue + Type: 'AWS::SQS::Queue' + Properties: + FifoQueue: !Ref FifoQueue +Outputs: + QueueURL: + Description: URL of newly created SQS Queue + Value: !Ref SQSQueue + QueueARN: + Description: ARN of newly created SQS Queue + Value: !GetAtt + - SQSQueue + - Arn + QueueName: + Description: Name newly created SQS Queue + Value: !GetAtt + - SQSQueue + - QueueName + DeadLetterQueueURL: + Description: URL of newly created SQS Queue + Value: !If [ CreateDeadLetterQueue, !Ref MyDeadLetterQueue, "" ] + DeadLetterQueueARN: + Description: ARN of newly created SQS Queue + Value: !If [ CreateDeadLetterQueue, !GetAtt MyDeadLetterQueue.Arn, "" ] + DeadLetterQueueName: + Description: Name newly created SQS Queue + Value: !If [ CreateDeadLetterQueue, !GetAtt MyDeadLetterQueue.QueueName, "" ] diff --git a/examples/s3bucket.yaml b/examples/s3bucket.yaml index ba8dbc2c9..33cb98e20 100644 --- a/examples/s3bucket.yaml +++ b/examples/s3bucket.yaml @@ -1,9 +1,9 @@ apiVersion: operator.aws/v1alpha1 kind: S3Bucket metadata: - name: chrishein-test-bucket-104 + name: chrishein-test-bucket-109 spec: - bucketName: chrishein-test-bucket-name-104 + bucketName: chrishein-test-bucket-name-109 versioning: false logging: enabled: false diff --git a/examples/sqs.yaml b/examples/sqsqueue.yaml similarity index 100% rename from examples/sqs.yaml rename to examples/sqsqueue.yaml diff --git a/models/s3bucket.yaml b/models/s3bucket.yaml index 9f0337d75..0c535ef04 100644 --- a/models/s3bucket.yaml +++ b/models/s3bucket.yaml @@ -65,11 +65,19 @@ spec: BucketARN is the full Amazon ARN for the bucket created structKey: BucketARN templateKey: BucketArn - # additionalResources: - # services: - # - type: ExternalName - # externalName: "{{.Obj.Spec.BucketName}}.s3-{{.Config.Region}}.amazonaws.com" - # ports: - # - protocol: tcp - # port: 443 + additionalResources: + services: + - name: s3BucketSvc + type: ExternalName + externalName: "{{.Obj.Spec.BucketName}}.s3-{{.Config.Region}}.amazonaws.com" + ports: + - port: 443 + configMap: + - name: s3BucketCM + data: + bucketName: "{{.Obj.Spec.BucketName}}" + bucketARN: "{{.Obj.Output.BucketARN}}" + serviceName: "{{.Obj}}" + region: "{{.Config.Region}}" + bucketURL: "{{.Obj.Output.BucketName}}.s3-{{.Config.Region}}.amazonaws.com" diff --git a/pkg/helpers/service.go b/pkg/helpers/service.go index 63099125d..fc8a80f12 100644 --- a/pkg/helpers/service.go +++ b/pkg/helpers/service.go @@ -6,6 +6,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// Data wrapps the object that is needed for the services type Data struct { Obj interface{} Config *config.Config