Merge pull request #21 from serokell/rvem/retry-vault-login

Add vault login retries

.github/workflows/check.yml

@@ -3,7 +3,7 @@ on: push
 
 jobs:
   check:
-    runs-on: self-hosted
+    runs-on: [nix, self-hosted]
     steps:
       - uses: actions/checkout@v3
 

modules/options.nix

@@ -134,6 +134,17 @@ let
           '';
         };
 
+        loginRetries = mkOption {
+          type = with types; int;
+          default = 5;
+          description = ''
+            Number of attempts script will try to login into Vault.
+            This may be useful in case secrets service is restarted when internet
+            connection is not yet available. Sadly After=network-online.target
+            doesn't always guarantee that.
+          '';
+        };
+
         __toString = mkOption {
           default = _: "${cfg.outPrefix}/${name}";
           readOnly = true;

modules/script.nix

@@ -3,7 +3,7 @@ let
   inherit (scfg)
     environmentKey quoteEnvironmentValues
     environmentVariableNamePrefix extraScript
-    user group secretsKey secretsAreBase64;
+    user group secretsKey secretsAreBase64 loginRetries;
   inherit (lib) optionalString toUpper;
 
   secretsPath = "${cfg.outPrefix}/${name}";
@@ -18,9 +18,18 @@ in
   # Make sure we start from a clean slate
   rm -rf "${secretsPath}"
   mkdir -p "${secretsPath}"
+  max_retry="${toString loginRetries}"
+  counter="0"
 
+  set +e
   # Log into Vault using credentials from environmentFile
-  vaultOutput="$(vault write -format=json auth/approle/login role_id="$VAULT_ROLE_ID" secret_id=- <<< "$VAULT_SECRET_ID")"
+  until vaultOutput="$(vault write -format=json auth/approle/login role_id="$VAULT_ROLE_ID" secret_id=- <<< "$VAULT_SECRET_ID")"; do
+    echo "Failed to login into Vault, retrying"
+    sleep 5
+    [[ counter -eq $max_retry ]] && echo "Failed to login into Vault" && exit 1
+    ((counter++))
+  done
+  set -e
   jq '.auth.client_token = "redacted"' <<< "$vaultOutput"
   VAULT_TOKEN="$(jq -r '.auth.client_token' <<< "$vaultOutput")"
   export VAULT_TOKEN