Skip to content

Commit

Permalink
sweep: DIRACGrid#7759 RSS: add a DIRAC to GOCDB service type conversion
Browse files Browse the repository at this point in the history
  • Loading branch information
fstagni committed Sep 11, 2024
1 parent 42eb265 commit 5027310
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 39 deletions.
9 changes: 5 additions & 4 deletions src/DIRAC/Core/LCG/GOCDBClient.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,10 +356,11 @@ def _downTimeXMLParsing(self, dt, siteOrRes, entities=None, startDateMax=None):

try:
dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]] = elements
dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]][
"URL"
] = urls[0]
except Exception:
if urls:
dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]][
"URL"
] = urls[0]
except Exception as e:
try:
dtDict[
str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["HOSTNAME"]
Expand Down
74 changes: 39 additions & 35 deletions src/DIRAC/ResourceStatusSystem/Command/DowntimeCommand.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,24 @@
from DIRAC.ResourceStatusSystem.Command.Command import Command


# conversion from DIRAC resource type to GOCDB service type
diracToGOC_conversion = {
# Computing elements
"HTCondorCE": "org.opensciencegrid.htcondorce",
"AREX": "ARC-CE",
# FTS
"FTS3": "FTS",
"FTS": "FTS",
# Storage elements
"disk_srm": "srm",
"tape_srm": "srm.nearline",
"disk_root": "xrootd",
"tape_root": "wlcg.xrootd.tape",
"disk_https": "webdav",
"tape_https": "wlcg.webdav.tape",
}


class DowntimeCommand(Command):
"""
Downtime "master" Command or removed DTs.
Expand All @@ -32,15 +50,8 @@ class DowntimeCommand(Command):
def __init__(self, args=None, clients=None):
super().__init__(args, clients)

if "GOCDBClient" in self.apis:
self.gClient = self.apis["GOCDBClient"]
else:
self.gClient = GOCDBClient()

if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
self.gClient = self.apis.get("GOCDBClient", GOCDBClient())
self.rmClient = self.apis.get("ResourceManagementClient", ResourceManagementClient())

def _storeCommand(self, result):
"""
Expand Down Expand Up @@ -135,27 +146,25 @@ def _prepareCommand(self):
else:
elementName = gocSite["Value"]

# The DIRAC se names mean nothing on the grid, but their hosts do mean.
# The DIRAC se names mean nothing on the grid, but their hosts and service types do mean.
elif elementType == "StorageElement":
# for SRM and SRM only, we need to distinguish if it's tape or disk
# if it's not SRM, then gOCDBServiceType will be None (and we'll use them all)
# Get the SE object and its protocols
try:
se = StorageElement(elementName)
seOptions = se.options
seProtocols = set(se.localAccessProtocolList) | set(se.localWriteProtocolList)
except AttributeError: # Sometimes the SE can't be instantiated properly
se_protocols = list(se.localAccessProtocolList)
se_protocols.extend(x for x in se.localWriteProtocolList if x not in se_protocols)
except AttributeError:
self.log.error("Failure instantiating StorageElement object", elementName)
return S_ERROR("Failure instantiating StorageElement")
if "SEType" in seOptions and "srm" in seProtocols:
# Type should follow the convention TXDY
seType = seOptions["SEType"]
diskSE = re.search("D[1-9]", seType) is not None
tapeSE = re.search("T[1-9]", seType) is not None
if tapeSE:
gOCDBServiceType = "srm.nearline"
elif diskSE:
gOCDBServiceType = "srm"

# Determine the SE type and update gOCDBServiceType accordingly
se_type = se.options.get("SEType", "")
if re.search(r"D[1-9]", se_type):
gOCDBServiceType = diracToGOC_conversion[f"disk_{se_protocols[0]}"]
elif re.search(r"T[1-9]", se_type):
gOCDBServiceType = diracToGOC_conversion[f"tape_{se_protocols[0]}"]

# Get the SE hosts and return an error if none are found
res = getSEHosts(elementName)
if not res["OK"]:
return res
Expand All @@ -166,7 +175,7 @@ def _prepareCommand(self):
elementName = seHosts # in this case it will return a list, because there might be more than one host only

elif elementType in ["FTS", "FTS3"]:
gOCDBServiceType = "FTS"
gOCDBServiceType = diracToGOC_conversion[elementType]
# WARNING: this method presupposes that the server is an FTS3 type
gocSite = getGOCFTSName(elementName)
if not gocSite["OK"]:
Expand All @@ -182,10 +191,7 @@ def _prepareCommand(self):
ceType = gConfig.getValue(
cfgPath("Resources", "Sites", siteName.split(".")[0], siteName, "CEs", elementName, "CEType")
)
if ceType == "HTCondorCE":
gOCDBServiceType = "org.opensciencegrid.htcondorce"
elif ceType == "AREX":
gOCDBServiceType = "ARC-CE"
gOCDBServiceType = diracToGOC_conversion[ceType]

return S_OK((element, elementName, hours, gOCDBServiceType))

Expand Down Expand Up @@ -236,9 +242,8 @@ def doNew(self, masterParams=None):

# cleaning the Cache
if elementNames:
cleanRes = self._cleanCommand(element, elementNames)
if not cleanRes["OK"]:
return cleanRes
if not (res := self._cleanCommand(element, elementNames))["OK"]:
return res

uniformResult = []

Expand Down Expand Up @@ -270,9 +275,8 @@ def doNew(self, masterParams=None):

uniformResult.append(dt)

storeRes = self._storeCommand(uniformResult)
if not storeRes["OK"]:
return storeRes
if not (res := self._storeCommand(uniformResult))["OK"]:
return res

return S_OK()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
# Mock external libraries / modules not interesting for the unit test
seMock = mock.MagicMock()
seMock.options = {"SEType": "T0D1"}
seMock.localAccessProtocolList = ["https", "root"]
seMock.localWriteProtocolList = ["https"]

mock_GOCDBClient = mock.MagicMock()
mock_RMClient = mock.MagicMock()
Expand Down

0 comments on commit 5027310

Please sign in to comment.