test: added a workflow test with input data

src/DIRAC/WorkloadManagementSystem/Client/DownloadInputData.py

@@ -39,6 +39,7 @@ def __init__(self, argumentsDict):
         """Standard constructor"""
         self.name = COMPONENT_NAME
         self.log = gLogger.getSubLogger(self.name)
+        self.log.showHeaders(True)
         self.inputData = argumentsDict["InputData"]
         self.configuration = argumentsDict["Configuration"]
         # Warning: this contains not only the SEs but also the file metadata
@@ -56,6 +57,11 @@ def execute(self, dataToResolve=None):
         to leave room for any produced files.
         """
 
+        print("in execute")
+        gLogger.always("in execute, gLogger")
+        print(self.log)
+        self.log.always("in execute, self.log")
+
         # Define local configuration options present at every site
         localSESet = set(self.configuration["LocalSEList"])
 

tests/.dirac-ci-config.yaml

@@ -1,6 +1,6 @@
 config:
   DIRAC_CI_SETUP_SCRIPT: DIRAC/tests/Jenkins/dirac_ci.sh
-  PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 1 -S dirac-JenkinsSetup -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --cert --certLocation=/home/dirac/PilotInstallDIR/etc/grid-security --CVMFS_locations=/home/dirac/PilotInstallDIR -o diracInstallOnly --wnVO=vo --debug
+  PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 2 -S dirac-JenkinsSetup -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --cert --certLocation=/home/dirac/PilotInstallDIR/etc/grid-security --CVMFS_locations=/home/dirac/PilotInstallDIR -o diracInstallOnly --wnVO=vo --debug
   PILOT_JSON: "{
         \"timestamp\": \"2023-02-13T14:34:26.725499\",
         \"CEs\": {

tests/CI/install_client.sh

@@ -57,13 +57,30 @@ echo -e "*** $(date -u) **** Client INSTALLATION START ****\n"
 
 installDIRAC
 
-#-------------------------------------------------------------------------------#
-echo -e "*** $(date -u) **** Submit a job ****\n"
-
 echo -e "*** $(date -u)  Getting a non privileged user\n" |& tee -a clientTestOutputs.txt
 dirac-login -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt
 
+#-------------------------------------------------------------------------------#
+echo -e "*** $(date -u) **** Submit a job ****\n"
+
 echo -e '[\n    Arguments = "Hello World";\n    Executable = "echo";\n    Site = "DIRAC.Jenkins.ch";' > test.jdl
 echo "    JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test.jdl
 echo "]" >> test.jdl
-dirac-wms-job-submit test.jdl
+dirac-wms-job-submit test.jdl "${DEBUG}" |& tee -a clientTestOutputs.txt
+
+#-------------------------------------------------------------------------------#
+echo -e "*** $(date -u) **** add a file ****\n"
+
+echo "I like pizza!" > test_lfn.txt
+ls -l
+echo $?
+dirac-dms-add-file LFN:/vo/test_lfn.txt test_lfn.txt S3-DIRECT
+echo $?
+
+#-------------------------------------------------------------------------------#
+echo -e "*** $(date -u) **** Submit a job with an input ****\n"
+
+echo -e '[\n    Arguments = "Hello World";\n    Executable = "echo";\n    Site = "DIRAC.Jenkins.ch";\n    InputData = "/vo/test_lfn.txt";' > test_dl.jdl
+echo "    JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test_dl.jdl
+echo "]" >> test_dl.jdl
+dirac-wms-job-submit test_dl.jdl "${DEBUG}" |& tee -a clientTestOutputs.txt

tests/Integration/WorkloadManagementSystem/Test_JobDB.py

@@ -154,6 +154,7 @@ def test_getJobJDL_nonOriginal(jobDB: JobDB):
         CPUTime = 86400;
         Executable = "dirac-jobexec";
         InputData = "";
+        InputDataModule = "DIRAC.WorkloadManagementSystem.Client.DownloadInputData";
         InputSandbox =
             {{
                 "../../Integration/WorkloadManagementSystem/exe-script.py",

tests/Jenkins/dirac-cfg-update-server.py

@@ -127,6 +127,7 @@
 #           DIRAC.Jenkins.ch
 #           {
 #             SE = SE-1
+#             SE += S3-DIRECT
 #             CEs
 #             {
 #               jenkins.cern.ch
@@ -163,7 +164,7 @@
         print(res["Message"])
         sys.exit(1)
 
-csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/SE", "SE-1")
+csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/SE", "SE-1,S3-DIRECT")
 csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/CEs/jenkins.cern.ch/CEType", "Test")
 csAPI.setOption(
     "Resources/Sites/DIRAC/DIRAC.Jenkins.ch/CEs/jenkins.cern.ch/Queues/jenkins-queue_not_important/maxCPUTime", "200000"
@@ -435,6 +436,31 @@
 csAPI.setOption("Operations/Defaults/Services/Catalogs/CatalogList", "FileCatalog, TSCatalog, MultiVOFileCatalog")
 
 
+# Adding InputDataPolicy section of Operations
+# Operations
+# {
+#   Defaults
+#   {
+#     InputDataPolicy
+#     {
+#      Download = DIRAC.WorkloadManagementSystem.Client.DownloadInputData
+#      InputDataModule = DIRAC.WorkloadManagementSystem.Client.DownloadInputData
+#     }
+#   }
+# }
+
+res = csAPI.createSection("Operations/Defaults/InputDataPolicy")
+if not res["OK"]:
+    print(res["Message"])
+    sys.exit(1)
+csAPI.setOption(
+    "Operations/Defaults/InputDataPolicy/Download", "DIRAC.WorkloadManagementSystem.Client.DownloadInputData"
+)
+csAPI.setOption(
+    "Operations/Defaults/InputDataPolicy/InputDataModule", "DIRAC.WorkloadManagementSystem.Client.DownloadInputData"
+)
+
+
 # Adding DataManagement section of Operations
 # Operations
 # {

tests/Workflow/Integration/Test_UserJobs.py

@@ -3,22 +3,21 @@
     Can be automatized.
 """
 # pylint: disable=protected-access, wrong-import-position, invalid-name, missing-docstring
+import multiprocessing
 import os
 import sys
 import unittest
-import multiprocessing
 
 import DIRAC
 
 DIRAC.initialize(extra_config_files=["pilot.cfg"])  # Initialize configuration
 
 from DIRAC import gLogger, rootPath
+from DIRAC.Interfaces.API.Dirac import Dirac
+from DIRAC.Interfaces.API.Job import Job
 from DIRAC.tests.Utilities.IntegrationTest import IntegrationTest
 from DIRAC.tests.Utilities.utils import find_all
 
-from DIRAC.Interfaces.API.Job import Job
-from DIRAC.Interfaces.API.Dirac import Dirac
-
 
 class UserJobTestCase(IntegrationTest):
     """Base class for the UserJob test cases"""
@@ -154,6 +153,27 @@ def test_execute(self):
         self.assertTrue(res["OK"])
 
 
+class DownloadSuccess(UserJobTestCase):
+    def test_execute(self):
+        """Testing a job that downloads inputs"""
+
+        job = Job()
+
+        job.setName("dl-test")
+        job.setExecutable("/bin/ls", "-l")
+        job.setLogLevel("DEBUG")
+        try:
+            # This is the standard location in Jenkins
+            job.setInputSandbox(find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
+        except (IndexError, KeyError):
+            job.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
+        job.setConfigArgs("pilot.cfg")
+        job.setInputData("/vo/test_lfn.txt")
+        job.setInputDataPolicy("Download")
+        res = job.runLocal(self.d)
+        self.assertTrue(res["OK"])
+
+
 class MPSuccess(UserJobTestCase):
     def test_fixed(self):
         """this tests executes a job that requires exactly 4 processors"""
@@ -245,6 +265,7 @@ def test_min1(self):
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(HelloWorldSuccess))
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(HelloWorldPlusSuccess))
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(LSSuccess))
+    suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(DownloadSuccess))
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(MPSuccess))
     suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(MPSuccessMinMax))
     testResult = unittest.TextTestRunner(verbosity=2).run(suite)