From 1c2eecf099b76936b75e8502a11b9b825ce60e63 Mon Sep 17 00:00:00 2001 From: Janusz Martyniak Date: Tue, 23 Jan 2024 13:01:32 +0100 Subject: [PATCH 1/4] feat: add a flag to vary a remote logger buffer size --- Pilot/dirac-pilot.py | 2 +- Pilot/pilotTools.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Pilot/dirac-pilot.py b/Pilot/dirac-pilot.py index 712ed00f..001b819f 100644 --- a/Pilot/dirac-pilot.py +++ b/Pilot/dirac-pilot.py @@ -70,7 +70,7 @@ if not sys.stdin.isatty(): receivedContent = sys.stdin.read() log = RemoteLogger( - pilotParams.loggerURL, "Pilot", pilotUUID=pilotParams.pilotUUID, debugFlag=pilotParams.debugFlag + pilotParams.loggerURL, "Pilot", bufsize=pilotParams.loggerBufsize, pilotUUID=pilotParams.pilotUUID, debugFlag=pilotParams.debugFlag ) log.info("Remote logger activated") log.buffer.write(receivedContent) diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index dce65a86..80cca4a1 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -512,6 +512,7 @@ def __init__( isPilotLoggerOn=True, pilotUUID="unknown", flushInterval=10, + bufsize=1000, ): """ c'tor @@ -523,7 +524,7 @@ def __init__( self.pilotUUID = pilotUUID self.isPilotLoggerOn = isPilotLoggerOn sendToURL = partial(sendMessage, url, pilotUUID, "sendMessage") - self.buffer = FixedSizeBuffer(sendToURL, autoflush=flushInterval) + self.buffer = FixedSizeBuffer(sendToURL, bufsize=bufsize, autoflush=flushInterval) def debug(self, msg, header=True, sendPilotLog=False): super(RemoteLogger, self).debug(msg, header) @@ -584,7 +585,7 @@ class FixedSizeBuffer(object): Once it's full, a message is sent to a remote server and the buffer is renewed. """ - def __init__(self, senderFunc, bufsize=10, autoflush=10): + def __init__(self, senderFunc, bufsize=1000, autoflush=10): """ Constructor. @@ -723,6 +724,7 @@ def __init__(self, pilotParams, dummy=""): # URL present and the flag is set: isPilotLoggerOn = pilotParams.pilotLogging and (loggerURL is not None) interval = pilotParams.loggerTimerInterval + bufsize = pilotParams.loggerBufsize if not isPilotLoggerOn: self.log = Logger(self.__class__.__name__, debugFlag=self.debugFlag) @@ -734,6 +736,7 @@ def __init__(self, pilotParams, dummy=""): pilotUUID=pilotParams.pilotUUID, debugFlag=self.debugFlag, flushInterval=interval, + bufsize=bufsize, ) self.log.isPilotLoggerOn = isPilotLoggerOn @@ -919,6 +922,7 @@ def __init__(self): self.pilotLogging = False self.loggerURL = None self.loggerTimerInterval = 0 + self.loggerBufsize = 1000 self.pilotUUID = "unknown" self.modules = "" # see dirac-install "-m" option documentation self.userEnvVariables = "" # see dirac-install "--userEnvVariables" option documentation @@ -1215,6 +1219,8 @@ def __initJSON2(self): self.loggerURL = pilotOptions.get("RemoteLoggerURL") # logger buffer flush interval in seconds. self.loggerTimerInterval = int(pilotOptions.get("RemoteLoggerTimerInterval", self.loggerTimerInterval)) + # logger buffer size in lines: + self.loggerBufsize = max(1, int(pilotOptions.get("RemoteLoggerBufsize", self.loggerBufsize))) pilotLogLevel = pilotOptions.get("PilotLogLevel", "INFO") if pilotLogLevel.lower() == "debug": self.debugFlag = True @@ -1222,6 +1228,7 @@ def __initJSON2(self): self.log.debug("JSON: Remote logging URL: %s" % self.loggerURL) self.log.debug("JSON: Remote logging buffer flush interval in sec.(0: disabled): %s" % self.loggerTimerInterval) self.log.debug("JSON: Remote/local logging debug flag: %s" % self.debugFlag) + self.log.debug("JSON: Remote logging buffer size (lines): %s" % self.loggerBufsize) # CE type if present, then Defaults, otherwise as defined in the code: if "Commands" in pilotOptions: From 3c4cd7ec267f5e55abcbd570a3f47fab26ecc82a Mon Sep 17 00:00:00 2001 From: Janusz Martyniak Date: Tue, 23 Jan 2024 15:51:04 +0100 Subject: [PATCH 2/4] feat: only flush a non-empty buffer --- Pilot/pilotTools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index 80cca4a1..bdc4d6f2 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -650,7 +650,7 @@ def flush(self): :return: None :rtype: None """ - if not self.output.closed: + if not self.output.closed and self._nlines > 0: self.output.flush() buf = self.getValue() self.senderFunc(buf) From eda83f144a063fecef9c8d5668f89ff99ade0dfd Mon Sep 17 00:00:00 2001 From: Janusz Martyniak Date: Tue, 23 Jan 2024 17:34:40 +0100 Subject: [PATCH 3/4] feat: add a CE whitelist for remote logging --- Pilot/pilotTools.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index bdc4d6f2..17096fd0 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -1221,6 +1221,16 @@ def __initJSON2(self): self.loggerTimerInterval = int(pilotOptions.get("RemoteLoggerTimerInterval", self.loggerTimerInterval)) # logger buffer size in lines: self.loggerBufsize = max(1, int(pilotOptions.get("RemoteLoggerBufsize", self.loggerBufsize))) + # logger CE white list + loggerCEsWhiteList = pilotOptions.get("RemoteLoggerCEsWhiteList") + # restrict remote logging to whitelisted CEs ([] or None => no restriction) + self.log.debug("JSON: Remote logging CE white list: %s" % loggerCEsWhiteList) + if loggerCEsWhiteList is not None: + if not isinstance(loggerCEsWhiteList, list): + loggerCEsWhiteList = [elem.strip() for elem in loggerCEsWhiteList.split(",")] + if self.ceName not in loggerCEsWhiteList: + self.pilotLogging = False + self.log.debug("JSON: Remote logging disabled for this CE: %s" % self.ceName) pilotLogLevel = pilotOptions.get("PilotLogLevel", "INFO") if pilotLogLevel.lower() == "debug": self.debugFlag = True From 977b98b42c1e4b55c516ab7f4ed00a6092545460 Mon Sep 17 00:00:00 2001 From: Janusz Martyniak Date: Wed, 24 Jan 2024 09:46:07 +0100 Subject: [PATCH 4/4] feat: add CE whitelist to pilot_newSchema.json for dteam --- tests/CI/pilot_newSchema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index 8caa6f5a..ebf25331 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -126,6 +126,7 @@ "GenericPilotGroup": "dteam_pilot", "GenericPilotDN": "VAR_USERDN", "RemoteLogging": "True", + "RemoteLoggerCEsWhiteList": "jenkins.cern.ch, jenkins-mp-pool.cern.ch", "RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging", "RemoteLoggerTimerInterval": 0, "PilotLogLevel": "DEBUG",