From 6b81ba0de44de335fd0d126c857e9f2a4796b4ad Mon Sep 17 00:00:00 2001 From: Mark Yen Date: Thu, 1 Oct 2020 11:22:40 -0700 Subject: [PATCH 1/6] fix: chart: Set memory limit for CCDB key rotation We were failing in CI when attempting to rotate the CCDB keys; this turns out to be because were were getting OOMKilled. It appears that the job needs more than 128Mi to complete (as in, 192Mi appears to work; intermediate values were not tested). Set the limits for the process explicitly to hopefully fix CI. --- chart/config/resources.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chart/config/resources.yaml b/chart/config/resources.yaml index c8fd8398cb..a2c1277bc7 100644 --- a/chart/config/resources.yaml +++ b/chart/config/resources.yaml @@ -144,6 +144,9 @@ resources: log-cache: log-cache: 2048 nats: ~ + rotate-cc-database-key: + rotate_cc_database_key: + rotate: {memory: {limit: 512, request: 192}} router: 200 scheduler: cc_deployment_updater: 320 From 5a1259086f38a0284a2bb2edaee246697a751a0a Mon Sep 17 00:00:00 2001 From: Mark Yen Date: Thu, 1 Oct 2020 11:29:19 -0700 Subject: [PATCH 2/6] fix: chart: CCDB rotate: disable log sidecar The CCDB database key rotation job is a single-purpose job that doesn't require a log sidecar, as only one container has meaningful output, and it's not long-running. --- .../operations/instance_groups/rotate-cc-database-key.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 chart/assets/operations/instance_groups/rotate-cc-database-key.yaml diff --git a/chart/assets/operations/instance_groups/rotate-cc-database-key.yaml b/chart/assets/operations/instance_groups/rotate-cc-database-key.yaml new file mode 100644 index 0000000000..a572f5daac --- /dev/null +++ b/chart/assets/operations/instance_groups/rotate-cc-database-key.yaml @@ -0,0 +1,3 @@ +- path: /instance_groups/name=rotate-cc-database-key/env?/bosh/agent/settings/disable_log_sidecar + type: replace + value: true From 42215ed58d5198ed1aedd134c11d9c4ff2c48537 Mon Sep 17 00:00:00 2001 From: Vlad Iovanov Date: Fri, 2 Oct 2020 00:26:06 +0300 Subject: [PATCH 3/6] Don't error if the job has already been deleted --- testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh b/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh index a2f71db84e..033cd15d7e 100755 --- a/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh +++ b/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh @@ -85,7 +85,7 @@ wait_for_rotate_pod_to_end() { if [[ -n "${exit_code}" ]]; then echo " $(blue "Completed")" echo Terminating job - kubectl delete --namespace "${KUBECF_NAMESPACE}" "$(rotate_job_name)" + kubectl delete --ignore-not-found=true --namespace "${KUBECF_NAMESPACE}" "$(rotate_job_name)" # shellcheck disable=SC2005 echo "$(green "OK")" exit "${exit_code}" From 9864e56ab19c8513bd505d646aa497a725dbe655 Mon Sep 17 00:00:00 2001 From: Jan Dubois Date: Thu, 1 Oct 2020 16:28:44 -0700 Subject: [PATCH 4/6] Don't try to delete job if it is already gone --- testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh b/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh index 033cd15d7e..c270a093d9 100755 --- a/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh +++ b/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh @@ -85,7 +85,10 @@ wait_for_rotate_pod_to_end() { if [[ -n "${exit_code}" ]]; then echo " $(blue "Completed")" echo Terminating job - kubectl delete --ignore-not-found=true --namespace "${KUBECF_NAMESPACE}" "$(rotate_job_name)" + job_name="$(rotate_job_name)" + if [ -n "${job_name}" ]; then + kubectl delete --ignore-not-found=true --namespace "${KUBECF_NAMESPACE}" "${job_name}" + fi # shellcheck disable=SC2005 echo "$(green "OK")" exit "${exit_code}" From bcb6df97f80201b7f72b3d35f361342293a8b6c7 Mon Sep 17 00:00:00 2001 From: Vlad Iovanov Date: Fri, 2 Oct 2020 02:43:35 +0300 Subject: [PATCH 5/6] Don't try to cleanup the job --- testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh b/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh index c270a093d9..3ec5e102a6 100755 --- a/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh +++ b/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh @@ -84,11 +84,7 @@ wait_for_rotate_pod_to_end() { exit_code="$(kubectl get "${pod_name}" --namespace "${KUBECF_NAMESPACE}" --output "jsonpath=${jsonpath}")" if [[ -n "${exit_code}" ]]; then echo " $(blue "Completed")" - echo Terminating job - job_name="$(rotate_job_name)" - if [ -n "${job_name}" ]; then - kubectl delete --ignore-not-found=true --namespace "${KUBECF_NAMESPACE}" "${job_name}" - fi + # shellcheck disable=SC2005 echo "$(green "OK")" exit "${exit_code}" From b1734504d0121e0ac52233934997e21b4e34707b Mon Sep 17 00:00:00 2001 From: Jan Dubois Date: Thu, 1 Oct 2020 23:18:34 -0700 Subject: [PATCH 6/6] Log exit code to STDOUT --- testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh b/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh index 3ec5e102a6..310f133c7f 100755 --- a/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh +++ b/testing/ccdb_key_rotation/rotate-ccdb-keys-test.sh @@ -86,7 +86,7 @@ wait_for_rotate_pod_to_end() { echo " $(blue "Completed")" # shellcheck disable=SC2005 - echo "$(green "OK")" + echo "$(green "OK [${exit_code}]")" exit "${exit_code}" fi sleep 1