Merge pull request #7783 from fstagni/cherry-pick-2-235138036-integra…

…tion

[sweep:integration] RSS: add a DIRAC to GOCDB service type conversion

src/DIRAC/Core/LCG/GOCDBClient.py

@@ -10,6 +10,7 @@
 from xml.dom import minidom
 
 import requests
+
 from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.Core.Security.Locations import getCAsLocation
 
@@ -356,10 +357,11 @@ def _downTimeXMLParsing(self, dt, siteOrRes, entities=None, startDateMax=None):
 
             try:
                 dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]] = elements
-                dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]][
-                    "URL"
-                ] = urls[0]
-            except Exception:
+                if urls:
+                    dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]][
+                        "URL"
+                    ] = urls[0]
+            except Exception as e:
                 try:
                     dtDict[
                         str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["HOSTNAME"]

src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py

@@ -4,25 +4,42 @@
     GOCDB downtimes that are modified or deleted are also synced.
 """
 import re
-from urllib.error import URLError
 from datetime import datetime, timedelta
 from operator import itemgetter
+from urllib.error import URLError
 
-from DIRAC import S_OK, S_ERROR, gConfig
-from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
-from DIRAC.Core.Utilities.SiteSEMapping import getSEHosts, getStorageElementsHosts
+from DIRAC import S_ERROR, S_OK, gConfig
 from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
 from DIRAC.ConfigurationSystem.Client.Helpers.Resources import (
+    getCESiteMapping,
     getFTS3Servers,
+    getGOCFTSName,
     getGOCSiteName,
     getGOCSites,
-    getGOCFTSName,
-    getCESiteMapping,
 )
+from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
+from DIRAC.Core.Utilities.SiteSEMapping import getSEHosts, getStorageElementsHosts
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient
 from DIRAC.ResourceStatusSystem.Command.Command import Command
 
+# conversion from DIRAC resource type to GOCDB service type
+diracToGOC_conversion = {
+    # Computing elements
+    "HTCondorCE": "org.opensciencegrid.htcondorce",
+    "AREX": "ARC-CE",
+    # FTS
+    "FTS3": "FTS",
+    "FTS": "FTS",
+    # Storage elements
+    "disk_srm": "srm",
+    "tape_srm": "srm.nearline",
+    "disk_root": "xrootd",
+    "tape_root": "wlcg.xrootd.tape",
+    "disk_https": "webdav",
+    "tape_https": "wlcg.webdav.tape",
+}
+
 
 class DowntimeCommand(Command):
     """
@@ -32,15 +49,8 @@ class DowntimeCommand(Command):
     def __init__(self, args=None, clients=None):
         super().__init__(args, clients)
 
-        if "GOCDBClient" in self.apis:
-            self.gClient = self.apis["GOCDBClient"]
-        else:
-            self.gClient = GOCDBClient()
-
-        if "ResourceManagementClient" in self.apis:
-            self.rmClient = self.apis["ResourceManagementClient"]
-        else:
-            self.rmClient = ResourceManagementClient()
+        self.gClient = self.apis.get("GOCDBClient", GOCDBClient())
+        self.rmClient = self.apis.get("ResourceManagementClient", ResourceManagementClient())
 
     def _storeCommand(self, result):
         """
@@ -135,27 +145,25 @@ def _prepareCommand(self):
             else:
                 elementName = gocSite["Value"]
 
-        # The DIRAC se names mean nothing on the grid, but their hosts do mean.
+        # The DIRAC SE names mean nothing on the grid, but their hosts and service types do mean.
         elif elementType == "StorageElement":
-            # for SRM and SRM only, we need to distinguish if it's tape or disk
-            # if it's not SRM, then gOCDBServiceType will be None (and we'll use them all)
+            # Get the SE object and its protocols
             try:
                 se = StorageElement(elementName)
-                seOptions = se.options
-                seProtocols = set(se.localAccessProtocolList) | set(se.localWriteProtocolList)
-            except AttributeError:  # Sometimes the SE can't be instantiated properly
+                se_protocols = list(se.localAccessProtocolList)
+                se_protocols.extend(x for x in se.localWriteProtocolList if x not in se_protocols)
+            except AttributeError:
                 self.log.error("Failure instantiating StorageElement object", elementName)
                 return S_ERROR("Failure instantiating StorageElement")
-            if "SEType" in seOptions and "srm" in seProtocols:
-                # Type should follow the convention TXDY
-                seType = seOptions["SEType"]
-                diskSE = re.search("D[1-9]", seType) is not None
-                tapeSE = re.search("T[1-9]", seType) is not None
-                if tapeSE:
-                    gOCDBServiceType = "srm.nearline"
-                elif diskSE:
-                    gOCDBServiceType = "srm"
 
+            # Determine the SE type and update gOCDBServiceType accordingly
+            se_type = se.options.get("SEType", "")
+            if re.search(r"D[1-9]", se_type):
+                gOCDBServiceType = diracToGOC_conversion[f"disk_{se_protocols[0]}"]
+            elif re.search(r"T[1-9]", se_type):
+                gOCDBServiceType = diracToGOC_conversion[f"tape_{se_protocols[0]}"]
+
+            # Get the SE hosts and return an error if none are found
             res = getSEHosts(elementName)
             if not res["OK"]:
                 return res
@@ -166,7 +174,7 @@ def _prepareCommand(self):
             elementName = seHosts  # in this case it will return a list, because there might be more than one host only
 
         elif elementType in ["FTS", "FTS3"]:
-            gOCDBServiceType = "FTS"
+            gOCDBServiceType = diracToGOC_conversion[elementType]
             # WARNING: this method presupposes that the server is an FTS3 type
             gocSite = getGOCFTSName(elementName)
             if not gocSite["OK"]:
@@ -182,10 +190,7 @@ def _prepareCommand(self):
             ceType = gConfig.getValue(
                 cfgPath("Resources", "Sites", siteName.split(".")[0], siteName, "CEs", elementName, "CEType")
             )
-            if ceType == "HTCondorCE":
-                gOCDBServiceType = "org.opensciencegrid.htcondorce"
-            elif ceType == "AREX":
-                gOCDBServiceType = "ARC-CE"
+            gOCDBServiceType = diracToGOC_conversion[ceType]
 
         return S_OK((element, elementName, hours, gOCDBServiceType))
 
@@ -236,9 +241,8 @@ def doNew(self, masterParams=None):
 
         # cleaning the Cache
         if elementNames:
-            cleanRes = self._cleanCommand(element, elementNames)
-            if not cleanRes["OK"]:
-                return cleanRes
+            if not (res := self._cleanCommand(element, elementNames))["OK"]:
+                return res
 
         uniformResult = []
 
@@ -270,9 +274,8 @@ def doNew(self, masterParams=None):
 
             uniformResult.append(dt)
 
-        storeRes = self._storeCommand(uniformResult)
-        if not storeRes["OK"]:
-            return storeRes
+        if not (res := self._storeCommand(uniformResult))["OK"]:
+            return res
 
         return S_OK()
 

src/DIRAC/ResourceStatusSystem/Command/test/Test_RSS_Command_GOCDBStatusCommand.py

@@ -1,11 +1,11 @@
 """ Test_RSS_Command_GOCDBStatusCommand
 """
 from datetime import datetime, timedelta
-
 from unittest import mock
+
 import pytest
 
-from DIRAC import gLogger, S_OK
+from DIRAC import S_OK, gLogger
 from DIRAC.ResourceStatusSystem.Command.DowntimeCommand import DowntimeCommand
 
 """
@@ -16,6 +16,8 @@
 # Mock external libraries / modules not interesting for the unit test
 seMock = mock.MagicMock()
 seMock.options = {"SEType": "T0D1"}
+seMock.localAccessProtocolList = ["https", "root"]
+seMock.localWriteProtocolList = ["https"]
 
 mock_GOCDBClient = mock.MagicMock()
 mock_RMClient = mock.MagicMock()