feat(autossl) add certificate renewal cooloff period (#59)

From #46

Co-authored-by: Kamil Figiela <kamil.figiela@gmail.com>

README.md

@@ -173,6 +173,26 @@ end}),
 `domain_whitelist_callback` function is provided with a second argument,
 which indicates whether the certificate is about to be served on incoming HTTP request (false) or new certificate is about to be requested (true). This allows to use cached values on hot path (serving requests) while fetching fresh data from storage for new certificates. One may also implement different logic, e.g. do extra checks before requesting new cert.
 
+In case of certificate request failure one may want to prevent ACME client to request another certificate immediatelly. By default, the cooloff period it is set to 300 seconds (5 minutes). It may be customized with `failure_cooloff` or with `failure_cooloff_callback` function, e.g. to implement exponential backoff.
+
+```lua
+    failure_cooloff_callback = function(domain, count)
+      if count == 1 then
+        return 600 -- 10 minutes
+      elseif count == 2 then
+        return 1800 -- 30 minutes
+      elseif count == 3 then
+        return 3600 -- 1 hour
+      elseif count == 4 then
+        return 43200 -- 12 hours
+      elseif count == 5 then
+        return 43200 -- 12 hours
+      else
+        return 86400 -- 24 hours
+      end
+    end
+```
+
 ## tls-alpn-01 challenge
 
 tls-alpn-01 challenge is currently supported on Openresty `1.15.8.x`, `1.17.8.x` and `1.19.3.x`.
@@ -346,6 +366,10 @@ default_config = {
   domain_whitelist = nil,
   -- restrict registering new cert only with domain checked by this function
   domain_whitelist_callback = nil,
+  -- interval to wait before retrying after failed certificate request
+  failure_cooloff = 300,
+  -- function that returns interval to wait before retrying after failed certificate request
+  failure_cooloff_callback = nil,
   -- the threshold to renew a cert before it expires, in seconds
   renew_threshold = 7 * 86400,
   -- interval to check cert renewal, in seconds

lib/resty/acme/autossl.lua

@@ -38,6 +38,10 @@ local default_config = {
   domain_whitelist = nil,
   -- restrict registering new cert only with domain checked by this function
   domain_whitelist_callback = nil,
+  -- certificate failure cooloff period, in seconds
+  failure_cooloff = 300,
+  -- certificate failure cooloff function, receives domain name and attempt count, should return cooloff period in seconds
+  failure_cooloff_callback = nil,
   -- the threshold to renew a cert before it expires, in seconds
   renew_threshold = 7 * 86400,
   -- interval to check cert renewal, in seconds
@@ -58,6 +62,7 @@ local domain_pkeys = {}
 
 local domain_key_types, domain_key_types_count
 local domain_whitelist, domain_whitelist_callback
+local failure_cooloff_callback
 
 --[[
   certs_cache = {
@@ -75,6 +80,8 @@ local CERTS_LOCK_TTL = 300
 local update_cert_lock_key_prefix = "update_lock:"
 local domain_cache_key_prefix = "domain:"
 local account_private_key_prefix = "account_key:"
+local certificate_failure_lock_key_prefix = "failure_lock:"
+local certificate_failure_count_prefix = "failed_attempts:"
 
 -- get cert from storage
 local function get_certkey(domain, typ)
@@ -229,6 +236,18 @@ function AUTOSSL.update_cert(data)
     return "cert update is not allowed for domain " .. data.domain
   end
 
+
+  -- If its failed in the past and its still cooling down
+  -- we dont do anything right now
+  local failure_lock_key = certificate_failure_lock_key_prefix .. data.domain
+  local failure_lock, _ = AUTOSSL.storage:get(failure_lock_key)
+  if failure_lock then
+    local now = ngx.now()
+    local remaining = failure_lock - now
+    ngx.log(ngx.INFO, "failure lock key exists for another ", remaining, " seconds. Not updating ", data.domain, " right now")
+    return nil
+  end
+
   -- Note that we lock regardless of key types
   -- Let's encrypt tends to have a (undocumented?) behaviour that if
   -- you submit an order with different CSR while the previous order is still pending
@@ -243,6 +262,21 @@ function AUTOSSL.update_cert(data)
 
   err = update_cert_handler(data)
 
+  local failure_count_key = certificate_failure_count_prefix .. data.domain
+  if err then
+    local count_storage, _ = AUTOSSL.storage:get(failure_count_key)
+    local count = (count_storage or 0) + 1
+    AUTOSSL.storage:set(failure_count_key, count)
+    local cooloff = AUTOSSL.config.failure_cooloff
+    if failure_cooloff_callback then
+      cooloff = failure_cooloff_callback(domain, count)
+    end
+    local now = ngx.now()
+    AUTOSSL.storage:add(failure_lock_key, now + cooloff, cooloff)
+  else
+    AUTOSSL.storage:set(failure_count_key, 0)
+  end
+
   -- yes we don't release lock, but wait it to expire after negative cache is cleared
   return err
 end
@@ -347,6 +381,16 @@ function AUTOSSL.init(autossl_config, acme_config)
                       "security issues as all SNI will trigger a creation of certificate")
   end
 
+  failure_cooloff_callback = autossl_config.failure_cooloff_callback
+  if failure_cooloff_callback and type(failure_cooloff_callback) ~= "function" then
+    error("failure_cooloff_callback must be a function, got " .. type(failure_cooloff_callback))
+  end
+
+  if not autossl_config.failure_cooloff and not failure_cooloff_callback then
+    ngx.log(ngx.WARN, "neither failure_cooloff or failure_cooloff_callback is defined, ",
+                      "any certificate failure will not cooloff which may trigger ACME API limits")
+  end
+
   for _, typ in ipairs(domain_key_types) do
     if autossl_config.domain_key_paths[typ] then
       local domain_key_f, err = io.open(autossl_config.domain_key_paths[typ])