Merge pull request #904 from amazeeio/feature/baas-improvements

Improve BaaS integration:

docs/using_lagoon/lagoon_yml.md

@@ -135,8 +135,8 @@ As most of the time it is not desirable to run the same cronjobs across all envi
     * Just a friendly name for identifying what the cronjob will do
 * `schedule:`
     * The schedule at which to execute the cronjob. This follows the standard convention of cron. If you're not sure about the syntax [Crontab Generator](https://crontab-generator.org/) can help.
-    * You can specify `H` for the minute, and your cronjob will run once per hour at a random minute (the same minute each hour), or `H/15` to run it every 15 mins but with a random offset from the hour (like `6,21,36,51`)
-    * You can specify `H` for the hour, and your cronjob will run once per day at a random hour (the same hour every day)
+    * You can specify `M` for the minute, and your cronjob will run once per hour at a random minute (the same minute each hour), or `M/15` to run it every 15 mins but with a random offset from the hour (like `6,21,36,51`)
+    * You can specify `H` for the hour, and your cronjob will run once per day at a random hour (the same hour every day) or `H(2-4)` to run it once per day within the hours of 2-4
 * `command:`
     * The command to execute. Like the tasks, this executes in the WORKDIR of the service, for Lagoon images this is `/app`
 * `service:`

helpers/annotate-pvc-backup.sh

@@ -7,4 +7,4 @@
 oc get pvc --all-namespaces | grep solr | sed '1d' | awk '{ print $2, "--namespace", $1 }' | while read line; do oc annotate --overwrite pvc $line appuio.ch/backup="true"; done
 oc get pvc --all-namespaces | grep nginx | sed '1d' | awk '{ print $2, "--namespace", $1 }' | while read line; do oc annotate --overwrite pvc $line appuio.ch/backup="true"; done
 
-oc get --all-namespaces pod -l 'service in (cli)' | sed '1d' | awk '{ print "--namespace", $1, $2 }'  | while read line; do oc annotate --overwrite pod $line appuio.ch/backupcommand='/bin/bash -c "if [[ $MARIADB_HOST ]]; then mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD $MARIADB_DATABASE; fi"'; done
+oc get --all-namespaces pod -l 'service in (cli)' | sed '1d' | awk '{ print "--namespace", $1, "pod", $2 }'  | while read line; do oc annotate $line --overwrite appuio.ch/backupcommand='/bin/sh -c "if [[ $MARIADB_HOST ]]; then dump=$(mktemp) && mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD $MARIADB_DATABASE > $dump && cat $dump && rm $dump; fi"'; done

images/oc-build-deploy-dind/build-deploy-docker-compose.sh

@@ -442,7 +442,7 @@ fi
 if oc get --insecure-skip-tls-verify customresourcedefinition schedules.backup.appuio.ch > /dev/null; then
   TEMPLATE_PARAMETERS=()
 
-  BACKUP_SCHEDULE=$( /oc-build-deploy/scripts/convert-crontab.sh "${OPENSHIFT_PROJECT}" "H 0 * * *")
+  BACKUP_SCHEDULE=$( /oc-build-deploy/scripts/convert-crontab.sh "${OPENSHIFT_PROJECT}" "M H(22-2) * * *")
   TEMPLATE_PARAMETERS+=(-p BACKUP_SCHEDULE="${BACKUP_SCHEDULE}")
 
   OPENSHIFT_TEMPLATE="/oc-build-deploy/openshift-templates/backup/schedule.yml"

images/oc-build-deploy-dind/openshift-templates/cli-persistent/deployment.yml

@@ -73,7 +73,7 @@ objects:
       template:
         metadata:
           annotations:
-            appuio.ch/backupcommand: /bin/sh -c "if [[ $MARIADB_HOST ]]; then mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD $MARIADB_DATABASE; fi"
+            appuio.ch/backupcommand: /bin/sh -c "if [[ $MARIADB_HOST ]]; then dump=$(mktemp) && mysqldump --max-allowed-packet=500M --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD $MARIADB_DATABASE > $dump && cat $dump && rm $dump; fi"
             backup.appuio.ch/file-extension: .mysql.sql
           creationTimestamp: null
           labels:

images/oc-build-deploy-dind/openshift-templates/mariadb-galera/cronjobs.yml

@@ -1,3 +1,3 @@
 - name: mariadb backup
-  schedule: "H 1 * * *"
+  schedule: "M 1 * * *"
   command: /lagoon/mysql-backup.sh 127.0.0.1

images/oc-build-deploy-dind/openshift-templates/mariadb-single/cronjobs.yml

@@ -1,3 +1,3 @@
 - name: backup
-  schedule: "H 1 * * *"
+  schedule: "M 1 * * *"
   command: /lagoon/mysql-backup.sh 127.0.0.1

images/oc-build-deploy-dind/openshift-templates/postgres/cronjobs.yml

@@ -1,3 +1,3 @@
 - name: backup
-  schedule: "H 1 * * *"
+  schedule: "M 1 * * *"
   command: /lagoon/postgres-backup.sh localhost

images/oc-build-deploy-dind/scripts/convert-crontab.sh

@@ -15,12 +15,12 @@ do
 
   # Minutes
   if [ "$index" = "0" ]; then
-    if [[ $piece =~ ^H$ ]]; then
-      # If just an H is defined, we generate a random minute
+    if [[ $piece =~ ^(M|H)$ ]]; then
+      # If just an M or H (for backwards compatibility) is defined, we generate a random minute
       MINUTES=$((SEED % 59))
 
-    elif [[ $piece =~ ^(H|\*)\/([0-5]?[0-9])$ ]]; then
-      # A Minute like H/15 or (*/15 for backwards compatibility) is defined, create a list of minutes with a random start
+    elif [[ $piece =~ ^(M|H|\*)\/([0-5]?[0-9])$ ]]; then
+      # A Minute like M/15 (or H/15 or */15 for backwards compatibility) is defined, create a list of minutes with a random start
       # like 4,19,34,49 or 6,21,36,51
       STEP=${BASH_REMATCH[2]}
       # Generate a random start within the given step to prevent that all cronjobs start at the same time
@@ -49,6 +49,33 @@ do
     if [[ $piece =~ ^H$ ]]; then
       # If just an H is defined, we generate a random hour
       HOURS=$((SEED % 23))
+    elif [[ $piece =~ ^H\(([01]?[0-9]|2[0-3])-([01]?[0-9]|2[0-3])\)$ ]]; then
+      # If H is defined with a given range, example: H(2-4), we generate a random hour between 2-4
+      HOUR_FROM=${BASH_REMATCH[1]}
+      HOUR_TO=${BASH_REMATCH[2]}
+      if (( HOUR_FROM < HOUR_TO )); then
+        # Example: HOUR_FROM: 2, HOUR_TO: 4
+        # Calculate the difference between the two hours (in example will be 2)
+        MAX_DIFFERENCE=$((HOUR_TO - HOUR_FROM))
+        # Generate a difference based on the SEED (in example will be 0, 1 or 2)
+        DIFFERENCE=$((SEED % MAX_DIFFERENCE))
+        # Add the generated difference to the FROM hour (in example will be 2, 3 or 4)
+        HOURS=$((HOUR_FROM + DIFFERENCE))
+      elif (( HOUR_FROM > HOUR_TO )); then
+        # If the FROM is larger than the TO, we have a range like 22-2
+        # Calculate the difference between the two hours with a 24 hour jump (in example will be 4)
+        MAX_DIFFERENCE=$((24 - HOUR_FROM + HOUR_TO))
+        # Generate a difference based on the SEED (in example will be 0, 1, 2, 3 or 4)
+        DIFFERENCE=$((SEED % MAX_DIFFERENCE))
+        # Add the generated difference to the FROM hour (in example will be 22, 23, 24, 25 or 26)
+        HOURS=$((HOUR_FROM + DIFFERENCE))
+        if (( HOURS >= 24 )); then
+          # If the hour is higher than 24, we subtract 24 to handle the midnight change
+          HOURS=$((HOURS - 24))
+        fi
+      elif (( HOUR_FROM = HOUR_TO )); then
+        HOURS=$HOUR_FROM
+      fi
     else
       HOURS=$piece
     fi