From 590a0bf94b68d1b3a98e0c34d5c281487af06987 Mon Sep 17 00:00:00 2001 From: Tammer Saleh Date: Wed, 29 Jan 2020 00:24:58 +0000 Subject: [PATCH 1/2] Upgrade Postgres and AWS instances AWS no longer allows provisioning Postgres v9.4, and they deprecated the m3 instance family. While fixing this, we also move the non CUI information from the secrets variables into inline pipeline.yml values. Putting DB versions and other non-CUI information in the variables files means we have to treat them as secrets, and means we don't get all of the git-nicities when modifying them. Finally, this commit fixes the development smoke tests, which were missing the `aws-db-test` repo. --- catalog-template.yml | 22 +++--- catalog-test.yml | 6 +- ci/pipeline.yml | 118 +++++++++++++++------------- ci/terraform/rds-internal.tf | 2 +- ci/terraform/rds-shared-postgres.tf | 2 +- ci/terraform/variables.tf | 2 + 6 files changed, 81 insertions(+), 71 deletions(-) diff --git a/catalog-template.yml b/catalog-template.yml index ebc4457f..cd60780b 100644 --- a/catalog-template.yml +++ b/catalog-template.yml @@ -12,7 +12,7 @@ rds: longDescription: providerDisplayName: RDS documentationUrl: "https://cloud.gov/docs/services/relational-database/" - supportUrl: + supportUrl: plans: - id: "1bbd9c4f-adb8-43dc-bbec-ab0315bcb14e" name: "shared-psql" @@ -107,7 +107,7 @@ rds: displayName: "Dedicated large PostgreSQL" free: false adapter: dedicated - instanceClass: db.m3.large + instanceClass: db.m4.large allocatedStorage: 10 dbType: postgres redundant: false @@ -135,7 +135,7 @@ rds: displayName: "Dedicated redundant large PostgreSQL" free: false adapter: dedicated - instanceClass: db.m3.large + instanceClass: db.m4.large allocatedStorage: 10 dbType: postgres redundant: true @@ -163,7 +163,7 @@ rds: displayName: "Dedicated x-large PostgreSQL" free: false adapter: dedicated - instanceClass: db.m3.xlarge + instanceClass: db.m4.xlarge allocatedStorage: 10 dbType: postgres redundant: false @@ -191,7 +191,7 @@ rds: displayName: "Dedicated redundant x-large PostgreSQL" free: false adapter: dedicated - instanceClass: db.m3.xlarge + instanceClass: db.m4.xlarge allocatedStorage: 10 dbType: postgres redundant: true @@ -299,7 +299,7 @@ rds: displayName: "Dedicated large MySQL" free: false adapter: dedicated - instanceClass: db.m3.large + instanceClass: db.m4.large allocatedStorage: 10 dbType: mysql dbVersion: *mysql-version @@ -328,7 +328,7 @@ rds: displayName: "Dedicated redundant large MySQL" free: false adapter: dedicated - instanceClass: db.m3.large + instanceClass: db.m4.large allocatedStorage: 10 dbType: mysql dbVersion: *mysql-version @@ -357,7 +357,7 @@ rds: displayName: "Dedicated x-large MySQL" free: false adapter: dedicated - instanceClass: db.m3.xlarge + instanceClass: db.m4.xlarge allocatedStorage: 10 dbType: mysql dbVersion: *mysql-version @@ -386,7 +386,7 @@ rds: displayName: "Dedicated redundant x-large MySQL" free: false adapter: dedicated - instanceClass: db.m3.xlarge + instanceClass: db.m4.xlarge allocatedStorage: 10 dbType: mysql dbVersion: *mysql-version @@ -415,7 +415,7 @@ rds: displayName: "Dedicated medium Oracle SE2" free: false adapter: dedicated - instanceClass: db.m3.medium + instanceClass: db.t3.medium allocatedStorage: 10 dbType: oracle-se2 licenseModel: license-included @@ -444,7 +444,7 @@ rds: displayName: "Dedicated large SQL Server 2016 SE" free: false adapter: dedicated - instanceClass: db.m3.large + instanceClass: db.m5.large allocatedStorage: 10 dbType: sqlserver-se licenseModel: license-included diff --git a/catalog-test.yml b/catalog-test.yml index 4735e7b9..04e8ad96 100644 --- a/catalog-test.yml +++ b/catalog-test.yml @@ -81,7 +81,7 @@ rds: displayName: "Dedicated Medium Postgres" free: false adapter: dedicated - instanceClass: db.m3.medium + instanceClass: db.t3.medium dbType: postgres securityGroup: sg-123456 subnetGroup: subnet-group @@ -152,7 +152,7 @@ rds: displayName: "Dedicated Medium MySQL" free: false adapter: dedicated - instanceClass: db.m3.medium + instanceClass: db.t3.medium dbType: mysql securityGroup: sg-123456 subnetGroup: subnet-group @@ -177,7 +177,7 @@ rds: displayName: "Dedicated Medium Oracle SE2" free: false adapter: dedicated - instanceClass: db.m3.medium + instanceClass: db.t3.medium dbType: oracle-se2 licenseModel: license-included securityGroup: sg-123456 diff --git a/ci/pipeline.yml b/ci/pipeline.yml index ce785d70..909e8006 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -27,27 +27,29 @@ jobs: TF_VAR_stack_description: ((development-stack-name)) TF_VAR_remote_state_bucket: ((development-s3-tfstate-bucket)) - TF_VAR_rds_internal_instance_type: ((development-rds-internal-instance-type)) - TF_VAR_rds_internal_db_size: ((development-rds-internal-db-size)) + TF_VAR_rds_internal_instance_type: db.m3.medium + TF_VAR_rds_internal_db_size: 10 TF_VAR_rds_internal_db_name: ((development-rds-internal-db-name)) - TF_VAR_rds_internal_db_engine: ((development-rds-internal-db-engine)) - TF_VAR_rds_internal_db_engine_version: ((development-rds-internal-db-engine-version)) + TF_VAR_rds_internal_db_engine: postgres + TF_VAR_rds_internal_db_engine_version: 9.4.20 + TF_VAR_rds_internal_db_parameter_group_family: postgres9.4 TF_VAR_rds_internal_username: ((development-rds-internal-username)) TF_VAR_rds_internal_password: ((development-rds-internal-password)) - TF_VAR_rds_shared_mysql_instance_type: ((development-rds-shared-mysql-instance-type)) - TF_VAR_rds_shared_mysql_db_size: ((development-rds-shared-mysql-db-size)) + TF_VAR_rds_shared_mysql_instance_type: db.m4.large + TF_VAR_rds_shared_mysql_db_size: 100 TF_VAR_rds_shared_mysql_db_name: ((development-rds-shared-mysql-db-name)) - TF_VAR_rds_shared_mysql_db_engine: ((development-rds-shared-mysql-db-engine)) - TF_VAR_rds_shared_mysql_db_engine_version: ((development-rds-shared-mysql-db-engine-version)) + TF_VAR_rds_shared_mysql_db_engine: mysql + TF_VAR_rds_shared_mysql_db_engine_version: 5.6.41 TF_VAR_rds_shared_mysql_username: ((development-rds-shared-mysql-username)) TF_VAR_rds_shared_mysql_password: ((development-rds-shared-mysql-password)) - TF_VAR_rds_shared_postgres_instance_type: ((development-rds-shared-postgres-instance-type)) - TF_VAR_rds_shared_postgres_db_size: ((development-rds-shared-postgres-db-size)) + TF_VAR_rds_shared_postgres_instance_type: db.m4.large + TF_VAR_rds_shared_postgres_db_size: 100 TF_VAR_rds_shared_postgres_db_name: ((development-rds-shared-postgres-db-name)) - TF_VAR_rds_shared_postgres_db_engine: ((development-rds-shared-postgres-db-engine)) - TF_VAR_rds_shared_postgres_db_engine_version: ((development-rds-shared-postgres-db-engine-version)) + TF_VAR_rds_shared_postgres_db_engine: postgres + TF_VAR_rds_shared_postgres_db_engine_version: 9.5.15 + TF_VAR_rds_shared_postgres_db_parameter_group_family: postgres9.5 TF_VAR_rds_shared_postgres_username: ((development-rds-shared-postgres-username)) TF_VAR_rds_shared_postgres_password: ((development-rds-shared-postgres-password)) @@ -83,10 +85,10 @@ jobs: AUTH_PASS: ((development-auth-pass)) AUTH_USER: ((development-auth-user)) AWS_DEFAULT_REGION: ((development-aws-default-region)) - DB_SSLMODE: ((development-rds-db-sslmode)) + DB_SSLMODE: require DB_USER: ((development-rds-internal-username)) DB_PASS: ((development-rds-internal-password)) - DB_TYPE: ((development-rds-internal-db-engine)) + DB_TYPE: postgres DB_NAME: ((development-rds-internal-db-name)) ENC_KEY: ((development-enc-key)) @@ -138,10 +140,12 @@ jobs: - name: acceptance-tests-development plan: - - get: aws-broker-app - resource: aws-broker-app-development - passed: [deploy-aws-broker-development] - trigger: true + - in_parallel: + - get: aws-broker-app + resource: aws-broker-app-development + passed: [deploy-aws-broker-development] + trigger: true + - get: aws-db-test - in_parallel: steps: - task: smoke-tests-postgres @@ -243,27 +247,29 @@ jobs: TF_VAR_stack_description: ((staging-stack-name)) TF_VAR_remote_state_bucket: ((staging-s3-tfstate-bucket)) - TF_VAR_rds_internal_instance_type: ((staging-rds-internal-instance-type)) - TF_VAR_rds_internal_db_size: ((staging-rds-internal-db-size)) + TF_VAR_rds_internal_instance_type: db.m3.medium + TF_VAR_rds_internal_db_size: 10 TF_VAR_rds_internal_db_name: ((staging-rds-internal-db-name)) - TF_VAR_rds_internal_db_engine: ((staging-rds-internal-db-engine)) - TF_VAR_rds_internal_db_engine_version: ((staging-rds-internal-db-engine-version)) + TF_VAR_rds_internal_db_engine: postgres + TF_VAR_rds_internal_db_engine_version: 9.4.20 + TF_VAR_rds_internal_db_parameter_group_family: postgres9.4 TF_VAR_rds_internal_username: ((staging-rds-internal-username)) TF_VAR_rds_internal_password: ((staging-rds-internal-password)) - TF_VAR_rds_shared_mysql_instance_type: ((staging-rds-shared-mysql-instance-type)) - TF_VAR_rds_shared_mysql_db_size: ((staging-rds-shared-mysql-db-size)) + TF_VAR_rds_shared_mysql_instance_type: db.m4.large + TF_VAR_rds_shared_mysql_db_size: 100 TF_VAR_rds_shared_mysql_db_name: ((staging-rds-shared-mysql-db-name)) - TF_VAR_rds_shared_mysql_db_engine: ((staging-rds-shared-mysql-db-engine)) - TF_VAR_rds_shared_mysql_db_engine_version: ((staging-rds-shared-mysql-db-engine-version)) + TF_VAR_rds_shared_mysql_db_engine: mysql + TF_VAR_rds_shared_mysql_db_engine_version: 5.6.41 TF_VAR_rds_shared_mysql_username: ((staging-rds-shared-mysql-username)) TF_VAR_rds_shared_mysql_password: ((staging-rds-shared-mysql-password)) - TF_VAR_rds_shared_postgres_instance_type: ((staging-rds-shared-postgres-instance-type)) - TF_VAR_rds_shared_postgres_db_size: ((staging-rds-shared-postgres-db-size)) + TF_VAR_rds_shared_postgres_instance_type: db.m4.large + TF_VAR_rds_shared_postgres_db_size: 100 TF_VAR_rds_shared_postgres_db_name: ((staging-rds-shared-postgres-db-name)) - TF_VAR_rds_shared_postgres_db_engine: ((staging-rds-shared-postgres-db-engine)) - TF_VAR_rds_shared_postgres_db_engine_version: ((staging-rds-shared-postgres-db-engine-version)) + TF_VAR_rds_shared_postgres_db_engine: postgres + TF_VAR_rds_shared_postgres_db_engine_version: 9.4.20 + TF_VAR_rds_shared_postgres_db_parameter_group_family: postgres9.4 TF_VAR_rds_shared_postgres_username: ((staging-rds-shared-postgres-username)) TF_VAR_rds_shared_postgres_password: ((staging-rds-shared-postgres-password)) @@ -299,10 +305,10 @@ jobs: AUTH_PASS: ((staging-auth-pass)) AUTH_USER: ((staging-auth-user)) AWS_DEFAULT_REGION: ((staging-aws-default-region)) - DB_SSLMODE: ((staging-rds-db-sslmode)) + DB_SSLMODE: require DB_USER: ((staging-rds-internal-username)) DB_PASS: ((staging-rds-internal-password)) - DB_TYPE: ((staging-rds-internal-db-engine)) + DB_TYPE: postgres DB_NAME: ((staging-rds-internal-db-name)) ENC_KEY: ((staging-enc-key)) @@ -458,27 +464,29 @@ jobs: TF_VAR_stack_description: ((prod-stack-name)) TF_VAR_remote_state_bucket: ((prod-s3-tfstate-bucket)) - TF_VAR_rds_internal_instance_type: ((prod-rds-internal-instance-type)) - TF_VAR_rds_internal_db_size: ((prod-rds-internal-db-size)) + TF_VAR_rds_internal_instance_type: db.m3.medium + TF_VAR_rds_internal_db_size: 10 TF_VAR_rds_internal_db_name: ((prod-rds-internal-db-name)) - TF_VAR_rds_internal_db_engine: ((prod-rds-internal-db-engine)) - TF_VAR_rds_internal_db_engine_version: ((prod-rds-internal-db-engine-version)) + TF_VAR_rds_internal_db_engine: postgres + TF_VAR_rds_internal_db_engine_version: 9.4.20 + TF_VAR_rds_internal_db_parameter_group_family: postgres9.4 TF_VAR_rds_internal_username: ((prod-rds-internal-username)) TF_VAR_rds_internal_password: ((prod-rds-internal-password)) - TF_VAR_rds_shared_mysql_instance_type: ((prod-rds-shared-mysql-instance-type)) - TF_VAR_rds_shared_mysql_db_size: ((prod-rds-shared-mysql-db-size)) + TF_VAR_rds_shared_mysql_instance_type: db.m4.large + TF_VAR_rds_shared_mysql_db_size: 100 TF_VAR_rds_shared_mysql_db_name: ((prod-rds-shared-mysql-db-name)) - TF_VAR_rds_shared_mysql_db_engine: ((prod-rds-shared-mysql-db-engine)) - TF_VAR_rds_shared_mysql_db_engine_version: ((prod-rds-shared-mysql-db-engine-version)) + TF_VAR_rds_shared_mysql_db_engine: mysql + TF_VAR_rds_shared_mysql_db_engine_version: 5.6.41 TF_VAR_rds_shared_mysql_username: ((prod-rds-shared-mysql-username)) TF_VAR_rds_shared_mysql_password: ((prod-rds-shared-mysql-password)) - TF_VAR_rds_shared_postgres_instance_type: ((prod-rds-shared-postgres-instance-type)) - TF_VAR_rds_shared_postgres_db_size: ((prod-rds-shared-postgres-db-size)) + TF_VAR_rds_shared_postgres_instance_type: db.m4.large + TF_VAR_rds_shared_postgres_db_size: 100 TF_VAR_rds_shared_postgres_db_name: ((prod-rds-shared-postgres-db-name)) - TF_VAR_rds_shared_postgres_db_engine: ((prod-rds-shared-postgres-db-engine)) - TF_VAR_rds_shared_postgres_db_engine_version: ((prod-rds-shared-postgres-db-engine-version)) + TF_VAR_rds_shared_postgres_db_engine: postgres + TF_VAR_rds_shared_postgres_db_engine_version: 9.4.20 + TF_VAR_rds_shared_postgres_db_parameter_group_family: postgres9.4 TF_VAR_rds_shared_postgres_username: ((prod-rds-shared-postgres-username)) TF_VAR_rds_shared_postgres_password: ((prod-rds-shared-postgres-password)) @@ -514,10 +522,10 @@ jobs: AUTH_PASS: ((prod-auth-pass)) AUTH_USER: ((prod-auth-user)) AWS_DEFAULT_REGION: ((prod-aws-default-region)) - DB_SSLMODE: ((prod-rds-db-sslmode)) + DB_SSLMODE: require DB_USER: ((prod-rds-internal-username)) DB_PASS: ((prod-rds-internal-password)) - DB_TYPE: ((prod-rds-internal-db-engine)) + DB_TYPE: postgres DB_NAME: ((prod-rds-internal-db-name)) ENC_KEY: ((prod-enc-key)) @@ -569,7 +577,7 @@ jobs: - name: acceptance-tests-prod plan: - - in_parallel: + - in_parallel: - get: aws-broker-app passed: [deploy-aws-broker-prod] trigger: true @@ -653,26 +661,26 @@ resources: - name: pipeline-tasks type: git source: - uri: ((pipeline-tasks-git-url)) - branch: ((pipeline-tasks-git-branch)) + uri: https://github.com/18F/cg-pipeline-tasks + branch: master - name: aws-broker-app type: git source: - uri: ((aws-broker-url)) - branch: ((aws-broker-branch)) + uri: https://github.com/18F/aws-broker + branch: master - name: aws-broker-app-development type: git source: - uri: ((aws-broker-url-development)) - branch: ((aws-broker-branch-development)) + uri: https://github.com/18F/aws-broker + branch: master - name: aws-db-test type: git source: - uri: ((aws-db-test-git-url)) - branch: ((aws-db-test-git-branch)) + uri: https://github.com/18F/cg-labratory + branch: master - name: deploy-aws-broker-development type: cf diff --git a/ci/terraform/rds-internal.tf b/ci/terraform/rds-internal.tf index 23d3eb40..05867ca0 100644 --- a/ci/terraform/rds-internal.tf +++ b/ci/terraform/rds-internal.tf @@ -13,5 +13,5 @@ module "rds_internal" { rds_db_engine_version = "${var.rds_internal_db_engine_version}" rds_username = "${var.rds_internal_username}" rds_password = "${var.rds_internal_password}" - rds_parameter_group_family = "postgres9.4" + rds_parameter_group_family = "${var.rds_internal_db_parameter_group_family}" } diff --git a/ci/terraform/rds-shared-postgres.tf b/ci/terraform/rds-shared-postgres.tf index fa73e945..a2a98a8b 100644 --- a/ci/terraform/rds-shared-postgres.tf +++ b/ci/terraform/rds-shared-postgres.tf @@ -13,5 +13,5 @@ module "rds_shared_postgres" { rds_db_engine_version = "${var.rds_shared_postgres_db_engine_version}" rds_username = "${var.rds_shared_postgres_username}" rds_password = "${var.rds_shared_postgres_password}" - rds_parameter_group_family = "postgres9.4" + rds_parameter_group_family = "${var.rds_shared_postgres_db_parameter_group_family}" } diff --git a/ci/terraform/variables.tf b/ci/terraform/variables.tf index 60986529..48eecc13 100644 --- a/ci/terraform/variables.tf +++ b/ci/terraform/variables.tf @@ -7,6 +7,7 @@ variable "rds_internal_db_size" {} variable "rds_internal_db_name" {} variable "rds_internal_db_engine" {} variable "rds_internal_db_engine_version" {} +variable "rds_internal_db_parameter_group_family" {} variable "rds_internal_username" {} variable "rds_internal_password" {} @@ -22,6 +23,7 @@ variable "rds_shared_postgres_instance_type" {} variable "rds_shared_postgres_db_size" {} variable "rds_shared_postgres_db_name" {} variable "rds_shared_postgres_db_engine" {} +variable "rds_shared_postgres_db_parameter_group_family" {} variable "rds_shared_postgres_db_engine_version" {} variable "rds_shared_postgres_username" {} variable "rds_shared_postgres_password" {} From 7f73da8dff8abc637b7dc7a1c2d54e4d81b9a4f4 Mon Sep 17 00:00:00 2001 From: Tammer Saleh Date: Mon, 3 Feb 2020 19:13:40 +0000 Subject: [PATCH 2/2] Keep git branch names outside of the pipeline.yml file This is more consistent with our other pipelines. --- ci/pipeline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 909e8006..1d743da7 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -662,25 +662,25 @@ resources: type: git source: uri: https://github.com/18F/cg-pipeline-tasks - branch: master + branch: ((pipeline-tasks-git-branch)) - name: aws-broker-app type: git source: uri: https://github.com/18F/aws-broker - branch: master + branch: ((aws-broker-branch)) - name: aws-broker-app-development type: git source: uri: https://github.com/18F/aws-broker - branch: master + branch: ((aws-broker-branch-development)) - name: aws-db-test type: git source: uri: https://github.com/18F/cg-labratory - branch: master + branch: ((aws-db-test-branch)) - name: deploy-aws-broker-development type: cf