From 8d0282cd1b5c3ce11ded5d35ee4cf941e28ebfc6 Mon Sep 17 00:00:00 2001 From: Joseph Huckaby Date: Tue, 28 Aug 2018 09:24:59 -0700 Subject: [PATCH] Version 0.8.26 Reversed course on new /api/app/status API call. It no longer causes a server restart on a tick age issue (i.e. broken timer). It now only logs a level 1 alert, and safely returns an API response. Server restarts should only be handled by an external monitoring tool. This tick age thing is an ongoing investigation into a very strange situtation, seen after about 20 days of continuous running on Linux CentOS 7.2, Node v10.6.0 and AWS (S3) storage backend. Seemingly at random (but on multiple servers at the same time), all the Node.js timers and intervals seem to cease firing, but all other actions triggered by sockets or child processes keep running. The server has to be restarted, sometimes SIGKILLed. CPU and memory are both fine. I'm currently dumbfounded as to what could cause such a thing. --- lib/api.js | 10 ++++++---- package.json | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/api.js b/lib/api.js index 32a17d9f..8b23b9e8 100644 --- a/lib/api.js +++ b/lib/api.js @@ -83,12 +83,14 @@ module.exports = Class.create({ callback(data); - // self-check: if tick_age is over 60 seconds, force a self restart + // self-check: if tick_age is over 60 seconds, log a level 1 debug alert if (tick_age > 60) { - var msg = "EMERGENCY SHUTDOWN: Tick age is over 60 seconds (" + Math.floor(tick_age) + "s)"; - this.logger.set('sync', true); + var msg = "EMERGENCY: Tick age is over 60 seconds (" + Math.floor(tick_age) + "s) -- Server should be restarted immediately."; this.logDebug(1, msg, data); - this.restartLocalServer({ reason: msg }); + + // JH 2018-08-28 Commenting this out for now, because an unsecured API should not have the power to cause an internal restart. + // This kind of thing should be handled by external monitoring tools. + // this.restartLocalServer({ reason: msg }); } }, diff --git a/package.json b/package.json index 18adc751..0df9acd1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "Cronicle", - "version": "0.8.25", + "version": "0.8.26", "description": "A simple, distributed task scheduler and runner with a web based UI.", "author": "Joseph Huckaby ", "homepage": "https://github.com/jhuckaby/Cronicle",