Merge pull request #4505 from zenoss/release/7.2.1

ZEN-34886: CZ 7.2.1 : Garfield Extended Fixes

Products/DataCollector/zenmodeler.py

@@ -58,7 +58,7 @@
 from Products.ZenEvents.ZenEventClasses import Heartbeat, Error
 from Products.ZenHub.PBDaemon import FakeRemote, PBDaemon, HubDown
 from Products.ZenUtils.Driver import drive, driveLater
-from Products.ZenUtils.Utils import unused, zenPath
+from Products.ZenUtils.Utils import unused, zenPath, wait
 from Products.Zuul.utils import safe_hasattr as hasattr
 
 # needed for Twisted's PB (Perspective Broker) to work
@@ -889,17 +889,23 @@ def heartbeat(self, ignored=None):
         # We start modeling from here to accomodate the startup delay.
 
         if not self.started:
-            if self.immediate == 0 and self.startat:
-                # This stuff relies on ARBITRARY_BEAT being < 60s
-                if self.timeMatches():
-                    self.started = True
-                    self.log.info("Starting modeling...")
-                    reactor.callLater(1, self.main)
+            if self.immediate == 0:
+                if self.startat:
+                    # This stuff relies on ARBITRARY_BEAT being < 60s
+                    if self.timeMatches():
+                        # Run modeling in case we have now=False, startat is not None and local time matches the startat
+                        self.started = True
+                        self.log.info("Starting modeling...")
+                        reactor.callLater(1, self.main)
                 elif not self.isMainScheduled:
+                    # Or run modeling by cycleTime in case we have now=False, startat is None
+                    # and we haven't set schedule by cycleTime yet
                     self.isMainScheduled = True
                     reactor.callLater(self.cycleTime(), self.main)
             else:
-                self.started = True
+                # Going back to the normal modeling schedule either cron or cycleTime
+                # after the first immediate modeling during service startup
+                self.immediate = 0
                 self.log.info(
                     "Starting modeling in %s seconds.", self.startDelay
                 )
@@ -941,6 +947,7 @@ def _devicegen_has_items(self):
                 self.devicegen = chain([first], self.devicegen)
         return result
 
+    @defer.inlineCallbacks
     def checkStop(self, unused=None):
         """
         Check to see if there's anything to do.
@@ -980,6 +987,11 @@ def checkStop(self, unused=None):
             if not self.options.cycle:
                 self.stop()
             self.finished = []
+            # frequency of heartbeat rate could be 2 times per minute in case we have
+            # cron job modeling faster than 1 minute it'll be trigger a second time
+            if runTime < 60 and self.startat is not None:
+                yield wait(60)
+            self.started = False
 
     def fillCollectionSlots(self, driver):
         """
@@ -1040,8 +1052,6 @@ def timeMatches(self):
         Check whether the current time matches a cron-like
         specification, return a straight true or false
         """
-        if self.startat is None:
-            return True
 
         def match_entity(entity, value):
             if entity == "*":
@@ -1333,7 +1343,7 @@ def mainLoop(self, driver):
         @return: Twisted deferred object
         @rtype: Twisted deferred object
         """
-        if self.options.cycle:
+        if self.options.cycle and self.startat is None:
             self.isMainScheduled = True
             driveLater(self.cycleTime(), self.mainLoop)
 

Products/Jobber/bin.py

@@ -7,7 +7,6 @@
 #
 ##############################################################################
 
-
 def main():
     import sys
 
@@ -16,6 +15,9 @@ def main():
     from celery.bin.celery import main
     from Products.ZenUtils.Utils import load_config
 
+    # work-around for celery's `--help` bug.
+    _print_help_when_requested()
+
     # Dynamic configuration shenanigans because Celery can't be re-configured
     # after its initial configuration has been set.
     _configure_celery()
@@ -28,11 +30,59 @@ def main():
     sys.exit(main())
 
 
+# Note: an empty tuple implies repetition of the key
+_import_names = {
+    "inspect": ("control", "inspect"),
+    "list": ("list", "list_"),
+    "report": ("celery", "report"),
+    "help": ("celery", "help"),
+}
+
+
+def _get_command(modname, cmdname):
+    import importlib
+
+    module = importlib.import_module("celery.bin.{}".format(modname))
+    return getattr(module, cmdname)
+
+
+def _print_help_when_requested():
+    import sys
+    from Products.Jobber.zenjobs import app
+
+    if "--help" not in sys.argv:
+        return
+
+    name = sys.argv[1]
+
+    if name == "--help":
+        sys.argv[1:] = ["help"]
+        return
+
+    if name == "monitor":
+        from Products.Jobber.monitor.command import MonitorCommand
+
+        w = MonitorCommand(app=app)
+        p = w.create_parser("zenjobs", "monitor")
+    else:
+        modname, cmdname = _import_names.get(sys.argv[1], (name, name))
+        command = _get_command(modname, cmdname)
+        cmd = command(app=app)
+        p = cmd.create_parser(sys.argv[0], name)
+
+    p.print_help()
+    sys.exit(0)
+
+
 def _configure_celery():
     import argparse
     import sys
     from Products.Jobber import config
 
+    # If '--help' was passed as an argument, don't attempt configuration.
+    if "--help" in sys.argv:
+        return
+
     parser = argparse.ArgumentParser()
     parser.add_argument("--config-file")
 
@@ -41,5 +91,5 @@ def _configure_celery():
         return
 
     cfg = config.getConfig(args.config_file)
-    config.ZenCeleryConfig = config.CeleryConfig.from_config(cfg)
+    config.ZenCeleryConfig = config.from_config(cfg)
     sys.argv[1:] = remainder

Products/Jobber/config.py

@@ -142,27 +142,28 @@ class CeleryConfig(object):
     CELERYBEAT_REDIRECT_STDOUTS = attr.ib(default=True)
     CELERYBEAT_REDIRECT_STDOUTS_LEVEL = attr.ib(default="INFO")
 
-    @classmethod
-    def from_config(cls, cfg={}):
-        args = {
-            "broker_url": buildBrokerUrl(cfg),
-            "result_backend": cfg.get("redis-url"),
-            "result_expires": cfg.get("zenjobs-job-expires"),
-            "worker_concurrency": cfg.get("concurrent-jobs"),
-            "worker_max_tasks_per_child": cfg.get("max-jobs-per-worker"),
-            "task_time_limit": cfg.get("job-hard-time-limit"),
-            "task_soft_time_limit": cfg.get("job-soft-time-limit"),
-            "beat_max_loop_interval": cfg.get(
-                "scheduler-max-loop-interval"
-            ),
-            "worker_proc_alive_timeout": cfg.get("zenjobs-worker-alive-timeout"),
-        }
-        tz = os.environ.get("TZ")
-        if tz:
-            args["timezone"] = tz
-
-        return cls(**args)
+
+def from_config(cfg=None):
+    cfg = cfg if cfg is not None else {}
+    args = {
+        "broker_url": buildBrokerUrl(cfg),
+        "result_backend": cfg.get("redis-url"),
+        "result_expires": cfg.get("zenjobs-job-expires"),
+        "worker_concurrency": cfg.get("concurrent-jobs"),
+        "worker_max_tasks_per_child": cfg.get("max-jobs-per-worker"),
+        "task_time_limit": cfg.get("job-hard-time-limit"),
+        "task_soft_time_limit": cfg.get("job-soft-time-limit"),
+        "beat_max_loop_interval": cfg.get(
+            "scheduler-max-loop-interval"
+        ),
+        "worker_proc_alive_timeout": cfg.get("zenjobs-worker-alive-timeout"),
+    }
+    tz = os.environ.get("TZ")
+    if tz:
+        args["timezone"] = tz
+
+    return CeleryConfig(**args)
 
 
 # Initialized with default values (for when --config-file is not specified)
-ZenCeleryConfig = CeleryConfig.from_config(getConfig())
+ZenCeleryConfig = from_config(getConfig())

Products/Jobber/log.py

@@ -419,7 +419,7 @@ def run(self):
 
 
 def _get_hash(config):
-    return hashlib.md5(
+    return hashlib.sha256(
         "".join("{0}{1}".format(k, config[k]) for k in sorted(config))
     ).hexdigest()
 

Products/Jobber/monitor/__init__.py

@@ -0,0 +1,12 @@
+##############################################################################
+#
+# Copyright (C) Zenoss, Inc. 2024, all rights reserved.
+#
+# This content is made available according to terms specified in
+# License.zenoss under the directory where your Zenoss product is installed.
+#
+##############################################################################
+
+from .command import MonitorCommand
+
+__all__ = ("MonitorCommand",)