diff --git a/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py b/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py index 9ba54b11e9c..6471ec65d0b 100644 --- a/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py +++ b/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py @@ -2,25 +2,14 @@ from unittest import mock import pytest -import tempfile -import errno -import DIRAC - -from DIRAC.tests.Utilities.utils import generateDIRACConfig -from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from diraccfg import CFG -from DIRAC.DataManagementSystem.private.FTS3Plugins.DefaultFTS3Plugin import DefaultFTS3Plugin +import DIRAC from DIRAC import S_OK from DIRAC.Core.Utilities.DErrno import cmpError - -from DIRAC.Resources.Storage.StorageBase import StorageBase - -from DIRAC.DataManagementSystem.Client.FTS3Job import FTS3Job from DIRAC.DataManagementSystem.Client.FTS3File import FTS3File -from DIRAC.DataManagementSystem.Client.FTS3Operation import FTS3Operation - +from DIRAC.DataManagementSystem.Client.FTS3Job import FTS3Job +from DIRAC.Resources.Storage.StorageBase import StorageBase +from DIRAC.tests.Utilities.utils import generateDIRACConfig DIRAC.gLogger.setLevel("DEBUG") # pylint: disable=redefined-outer-name diff --git a/src/DIRAC/DataManagementSystem/Client/test/Test_SE.py b/src/DIRAC/DataManagementSystem/Client/test/Test_SE.py deleted file mode 100644 index f2fe5fc3f25..00000000000 --- a/src/DIRAC/DataManagementSystem/Client/test/Test_SE.py +++ /dev/null @@ -1,308 +0,0 @@ -import itertools -import os -import tempfile -import pytest -from unittest import mock - -from diraccfg import CFG - -from DIRAC import S_OK -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient -from DIRAC.Resources.Storage.StorageBase import StorageBase -from DIRAC.Resources.Storage.StorageElement import StorageElementItem - - -class fake_SRM2Plugin(StorageBase): - def putFile(self, lfns, sourceSize=0): - return S_OK({"Successful": dict.fromkeys(lfns, "srm:putFile"), "Failed": {}}) - - def getTransportURL(self, path, protocols=False): - return S_OK({"Successful": dict.fromkeys(path, "srm:getTransportURL"), "Failed": {}}) - - -class fake_XROOTPlugin(StorageBase): - def putFile(self, lfns, sourceSize=0): - return S_OK({"Successful": dict.fromkeys(lfns, "root:putFile"), "Failed": {}}) - - def getTransportURL(self, path, protocols=False): - return S_OK({"Successful": dict.fromkeys(path, "root:getTransportURL"), "Failed": {}}) - - -class fake_GSIFTPPlugin(StorageBase): - def putFile(self, lfns, sourceSize=0): - return S_OK({"Successful": dict.fromkeys(lfns, "gsiftp:putFile"), "Failed": {}}) - - def getTransportURL(self, path, protocols=False): - return S_OK({"Successful": dict.fromkeys(path, "gsiftp:getTransportURL"), "Failed": {}}) - - -def mock_StorageFactory_generateStorageObject( - self, storageName, pluginName, parameters, hideExceptions=False -): # pylint: disable=unused-argument - """Generate fake storage object""" - storageObj = StorageBase(storageName, parameters) - storageObj.pluginName = pluginName - - return S_OK(storageObj) - -@pytest.fixture(scope="function") -def setup_environment(monkeypatch): - def mock_init(self, useProxy=False, vo=None): - self.proxy = useProxy - self.resourceStatus = mock.MagicMock() - self.vo = vo - self.remoteProtocolSections = [] - self.localProtocolSections = [] - self.name = "" - self.options = {} - self.protocols = {} - self.storages = {} - - monkeypatch.setattr( - "DIRAC.Resources.Storage.StorageFactory.StorageFactory.__init__", mock_init - ) - - monkeypatch.setattr( - "DIRAC.Resources.Storage.StorageFactory.StorageFactory._StorageFactory__generateStorageObject", - mock_StorageFactory_generateStorageObject - ) - - monkeypatch.setattr( - "DIRAC.Resources.Storage.StorageElement.StorageElementItem._StorageElementItem__isLocalSE", - lambda self: S_OK(True) - ) - - monkeypatch.setattr( - "DIRAC.Resources.Storage.StorageElement.StorageElementItem.addAccountingOperation", - lambda self: None - ) - - # Create test configuration file - testCfgFileName = os.path.join(tempfile.gettempdir(), "test_StorageElement.cfg") - cfgContent = """ - DIRAC - { - Setup=TestSetup - } - Resources{ - StorageElements{ - StorageA - { - BackendType = local - ReadAccess = Active - WriteAccess = Active - AccessProtocol.0 - { - Host = - PluginName = File - Protocol = file - Path = - } - } - StorageB - { - BackendType = local - ReadAccess = Active - WriteAccess = Active - AccessProtocol.0 - { - Host = - PluginName = SRM2 - Protocol = srm - Path = - } - } - StorageC - { - BackendType = local - ReadAccess = Active - WriteAccess = Active - AccessProtocol.0 - { - Host = - PluginName = XROOT - Protocol = root - Path = - } - } - StorageD - { - BackendType = local - ReadAccess = Active - WriteAccess = Active - AccessProtocol.0 - { - Host = - PluginName = SRM2 - Protocol = srm - Path = - } - AccessProtocol.1 - { - Host = - PluginName = XROOT - Protocol = root - Path = - } - } - StorageE - { - BackendType = local - ReadAccess = Active - WriteAccess = Active - WriteProtocols = root - WriteProtocols += srm - AccessProtocol.0 - { - Host = - PluginName = SRM2 - Protocol = srm - Path = - } - AccessProtocol.1 - { - Host = - PluginName = XROOT - Protocol = root - Path = - } - } - StorageX - { - BackendType = local - ReadAccess = Active - WriteAccess = Active - WriteProtocols = gsiftp - AccessProtocols = root - AccessProtocol.0 - { - Host = - PluginName = GSIFTP - Protocol = gsiftp - Path = - } - AccessProtocol.1 - { - Host = - PluginName = XROOT - Protocol = root - Path = - } - } - StorageY - { - BackendType = local - ReadAccess = Active - WriteAccess = Active - AccessProtocols = gsiftp - AccessProtocols += srm - AccessProtocol.0 - { - Host = - PluginName = GSIFTP - Protocol = gsiftp - Path = - } - AccessProtocol.1 - { - Host = - PluginName = SRM2 - Protocol = srm - Path = - } - } - StorageZ - { - BackendType = local - ReadAccess = Active - WriteAccess = Active - AccessProtocols = root - AccessProtocols += srm - WriteProtocols = root - WriteProtocols += srm - AccessProtocol.0 - { - Host = - PluginName = ROOT - Protocol = root - Path = - } - AccessProtocol.1 - { - Host = - PluginName = SRM2 - Protocol = srm - Path = - } - } - } - - } - Operations{ - Defaults - { - DataManagement{ - AccessProtocols = fakeProto - AccessProtocols += root - WriteProtocols = srm - } - } - } - """ - - with open(testCfgFileName, "w") as f: - f.write(cfgContent) - - gConfigurationData.localCFG = CFG() - gConfigurationData.remoteCFG = CFG() - gConfigurationData.mergedCFG = CFG() - gConfigurationData.generateNewVersion() - - gConfig = ConfigurationClient( - fileToLoadList=[testCfgFileName] - ) - - seA = StorageElementItem("StorageA") - seA.vo = "lhcb" - seB = StorageElementItem("StorageB") - seB.vo = "lhcb" - seC = StorageElementItem("StorageC") - seC.vo = "lhcb" - seD = StorageElementItem("StorageD") - seD.vo = "lhcb" - seE = StorageElementItem("StorageE") - seE.vo = "lhcb" - seX = StorageElementItem("StorageX") - seX.vo = "lhcb" - seY = StorageElementItem("StorageY") - seY.vo = "lhcb" - seZ = StorageElementItem("StorageZ") - seZ.vo = "lhcb" - - yield seA, seB, seC, seD, seE, seX, seY, seZ - - os.remove(testCfgFileName) - gConfigurationData.localCFG = CFG() - gConfigurationData.remoteCFG = CFG() - gConfigurationData.mergedCFG = CFG() - gConfigurationData.generateNewVersion() - - -@pytest.mark.usefixtures("setup_environment") -def test_negociateProtocolWithOtherSE(setup_environment): - seA, seB, seC, seD, seE, seX, seY, seZ = setup_environment - - res = seA.negociateProtocolWithOtherSE(seB) - assert res["OK"] - assert res["Value"] == ["file"] - - res = seB.negociateProtocolWithOtherSE(seA) - assert res["OK"] - assert res["Value"] == ["file"] - - res = seA.negociateProtocolWithOtherSE(seC) - assert res["OK"] - assert res["Value"] == [] - - res = seC.negociateProtocolWith diff --git a/src/DIRAC/DataManagementSystem/private/FTS3Plugins/test/Test_DefaultFTS3Plugin.py b/src/DIRAC/DataManagementSystem/private/FTS3Plugins/test/Test_DefaultFTS3Plugin.py index 286b14841e8..2762f622471 100644 --- a/src/DIRAC/DataManagementSystem/private/FTS3Plugins/test/Test_DefaultFTS3Plugin.py +++ b/src/DIRAC/DataManagementSystem/private/FTS3Plugins/test/Test_DefaultFTS3Plugin.py @@ -2,8 +2,8 @@ import os import tempfile +from unittest import mock -import mock import pytest from diraccfg import CFG @@ -343,10 +343,8 @@ def test_multiHop_specificLink(fts3Plugin): assert hopName == "CERN-RAW-CNAF-DST" -def test_multiHop_specificSrc_baseSEDst(mocker, fts3Plugin): +def test_multiHop_specificSrc_baseSEDst(fts3Plugin): """When we have a specific source and a baseSE dest""" - # mocker.patch("DIRAC.DataManagementSystem.private.FTS3Plugins.DefaultFTS3Plugin.StorageElement", return_value=mock.MagicMock()) - hopName = fts3Plugin.findMultiHopSEToCoverUpForWLCGFailure("CERN-RAW", "CNAF_MC-DST") assert hopName == "CERN-RAW-CNAF-Disk" diff --git a/src/DIRAC/Interfaces/API/DiracAdmin.py b/src/DIRAC/Interfaces/API/DiracAdmin.py index 214309800a1..51167069298 100755 --- a/src/DIRAC/Interfaces/API/DiracAdmin.py +++ b/src/DIRAC/Interfaces/API/DiracAdmin.py @@ -6,20 +6,19 @@ """ import os -from DIRAC import gLogger, gConfig, S_OK, S_ERROR -from DIRAC.Core.Utilities.PromptUser import promptUser -from DIRAC.Core.Base.API import API +from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup +from DIRAC.Core.Base.API import API from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager +from DIRAC.Core.Utilities.PromptUser import promptUser from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient +from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient -from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus from DIRAC.WorkloadManagementSystem.Client.JobManagerClient import JobManagerClient -from DIRAC.WorkloadManagementSystem.Client.WMSAdministratorClient import WMSAdministratorClient from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient +from DIRAC.WorkloadManagementSystem.Client.WMSAdministratorClient import WMSAdministratorClient voName = "" ret = getProxyInfo(disableVOMS=True) diff --git a/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py b/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py index bf259cfb8ea..22f5494279a 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py +++ b/src/DIRAC/ResourceStatusSystem/Client/ResourceStatus.py @@ -7,24 +7,23 @@ """ import math -from time import sleep from datetime import datetime, timedelta +from time import sleep -from DIRAC import gConfig, gLogger, S_OK, S_ERROR +from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient +from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import getPoliciesThatApply from DIRAC.ResourceStatusSystem.Utilities.RSSCacheNoThread import RSSCache from DIRAC.ResourceStatusSystem.Utilities.RssConfiguration import RssConfiguration -from DIRAC.ResourceStatusSystem.Utilities.InfoGetter import getPoliciesThatApply class ResourceStatus(metaclass=DIRACSingleton): """ - ResourceStatus helper that connects to CS if RSS flag is not Active. - It keeps the connection to the db / server as an object member, to avoid creating a new - one massively. + ResourceStatus helper keeps the connection to the db / server as an object member, + to avoid creating a new connection every time we need to do one. """ def __init__(self): diff --git a/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py b/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py index 6b103dee599..e789f5a7d19 100644 --- a/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py +++ b/src/DIRAC/ResourceStatusSystem/Client/SiteStatus.py @@ -7,17 +7,14 @@ import errno import math -from time import sleep from datetime import datetime, timedelta +from time import sleep -from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton -from DIRAC.Core.Utilities import DErrno +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.WorkloadManagementSystem.Client.WMSAdministratorClient import WMSAdministratorClient +from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient -from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus from DIRAC.ResourceStatusSystem.Utilities.RSSCacheNoThread import RSSCache from DIRAC.ResourceStatusSystem.Utilities.RssConfiguration import RssConfiguration diff --git a/src/DIRAC/Resources/Storage/test/Test_StorageElement.py b/src/DIRAC/Resources/Storage/test/Test_StorageElement.py index 0ca2318138e..90c9c2200f3 100644 --- a/src/DIRAC/Resources/Storage/test/Test_StorageElement.py +++ b/src/DIRAC/Resources/Storage/test/Test_StorageElement.py @@ -1,20 +1,18 @@ """ test StoragElement """ +import itertools import os import tempfile -from unittest import mock import unittest -import itertools +from unittest import mock from diraccfg import CFG from DIRAC import S_OK -from DIRAC.Resources.Storage.StorageElement import StorageElementItem -from DIRAC.Resources.Storage.StorageBase import StorageBase - - from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient +from DIRAC.Resources.Storage.StorageBase import StorageBase +from DIRAC.Resources.Storage.StorageElement import StorageElementItem class fake_SRM2Plugin(StorageBase): diff --git a/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py b/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py index 479359faab3..9f397b210e2 100755 --- a/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py +++ b/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py @@ -1,15 +1,8 @@ -import os -import tempfile import pytest - -from diraccfg import CFG import DIRAC -from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC import S_OK from DIRAC.Resources.Storage.StorageFactory import StorageFactory - from DIRAC.tests.Utilities.utils import generateDIRACConfig CFG_CONTENT = """