fix(backup): backups are not deleted when retained nr of backups >= 20

This commit introduces a new module variable: backup_runs_list_max_results
this variable is used to control the maximum amount of results that are
returned by the googleapis.sqladmin.v1.backupRuns.list API call.
By defaul this call only returns 20 results leading to a situation
where no old backups will be deleted whenever the desired amount of
retained backups exceed this number.

Fixes: #519

modules/backup/README.md

@@ -54,6 +54,7 @@ fetch workflows.googleapis.com/Workflow
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | backup\_retention\_time | The number of days backups should be kept | `number` | `30` | no |
+| backup\_runs\_list\_max\_results | The max amount of backups to list when fetching internal backup runs for the instance. This number must be larger then the amount of backups you wish to keep. E.g. for a daily backup schedule and a backup\_retention\_time of 30 days, you'd need to set this to at least 31 for old backups to get deleted. | `number` | `31` | no |
 | backup\_schedule | The cron schedule to execute the internal backup | `string` | `"45 2 * * *"` | no |
 | compress\_export | Whether or not to compress the export when storing in the bucket; Only valid for MySQL and PostgreSQL | `bool` | `true` | no |
 | connector\_params\_timeout | The end-to-end duration the connector call is allowed to run for before throwing a timeout exception. The default value is 1800 and this should be the maximum for connector methods that are not long-running operations. Otherwise, for long-running operations, the maximum timeout for a connector call is 31536000 seconds (one year). | `number` | `1800` | no |

modules/backup/main.tf

@@ -65,9 +65,10 @@ resource "google_workflows_workflow" "sql_backup" {
   project         = var.project_id
   service_account = local.service_account
   source_contents = templatefile("${path.module}/templates/backup.yaml.tftpl", {
-    project             = var.project_id
-    instanceName        = var.sql_instance
-    backupRetentionTime = var.backup_retention_time
+    project                  = var.project_id
+    instanceName             = var.sql_instance
+    backupRetentionTime      = var.backup_retention_time
+    backupRunsListMaxResults = var.backup_runs_list_max_results
   })
 }
 
@@ -105,7 +106,6 @@ resource "google_workflows_workflow" "sql_export" {
   source_contents = templatefile("${path.module}/templates/export.yaml.tftpl", {
     project                = var.project_id
     instanceName           = var.use_sql_instance_replica_in_exporter ? var.sql_instance_replica : var.sql_instance
-    backupRetentionTime    = var.backup_retention_time
     databases              = jsonencode(var.export_databases)
     gcsBucket              = var.export_uri
     dbType                 = split("_", data.google_sql_database_instance.backup_instance.database_version)[0]

modules/backup/templates/backup.yaml.tftpl

@@ -36,6 +36,7 @@ main:
       args:
         project: ${project}
         instance: ${instanceName}
+        maxResults: ${backupRunsListMaxResults}
       result: backupList
   - delete old backups:
       for:

modules/backup/variables.tf

@@ -42,6 +42,12 @@ variable "backup_retention_time" {
   default     = 30
 }
 
+variable "backup_runs_list_max_results" {
+  description = "The max amount of backups to list when fetching internal backup runs for the instance. This number must be larger then the amount of backups you wish to keep. E.g. for a daily backup schedule and a backup_retention_time of 30 days, you'd need to set this to at least 31 for old backups to get deleted."
+  type        = number
+  default     = 31
+}
+
 variable "scheduler_timezone" {
   description = "The Timezone in which the Scheduler Jobs are triggered"
   type        = string