From 3cf83e630526a082ed988ac07e86e64ffc3dcb87 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 23 May 2024 17:46:26 +0200 Subject: [PATCH] test: added a workflow test with input data --- tests/.dirac-ci-config.yaml | 2 +- tests/CI/install_client.sh | 24 ++++++++++++++--- tests/Jenkins/dirac-cfg-update-server.py | 3 ++- tests/Workflow/Integration/Test_UserJobs.py | 29 ++++++++++++++++++--- 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/tests/.dirac-ci-config.yaml b/tests/.dirac-ci-config.yaml index 126e8a12a65..42c53e67037 100644 --- a/tests/.dirac-ci-config.yaml +++ b/tests/.dirac-ci-config.yaml @@ -1,6 +1,6 @@ config: DIRAC_CI_SETUP_SCRIPT: DIRAC/tests/Jenkins/dirac_ci.sh - PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 1 -S dirac-JenkinsSetup -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --cert --certLocation=/home/dirac/PilotInstallDIR/etc/grid-security --CVMFS_locations=/home/dirac/PilotInstallDIR -o diracInstallOnly --wnVO=vo --debug + PILOT_INSTALLATION_COMMAND: dirac-pilot.py --modules /home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC -M 2 -S dirac-JenkinsSetup -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --pilotUUID=whatever12345 --cert --certLocation=/home/dirac/PilotInstallDIR/etc/grid-security --CVMFS_locations=/home/dirac/PilotInstallDIR -o diracInstallOnly --wnVO=vo --debug PILOT_JSON: "{ \"timestamp\": \"2023-02-13T14:34:26.725499\", \"CEs\": { diff --git a/tests/CI/install_client.sh b/tests/CI/install_client.sh index a02c6456e60..bfaab1827af 100755 --- a/tests/CI/install_client.sh +++ b/tests/CI/install_client.sh @@ -57,13 +57,29 @@ echo -e "*** $(date -u) **** Client INSTALLATION START ****\n" installDIRAC -#-------------------------------------------------------------------------------# -echo -e "*** $(date -u) **** Submit a job ****\n" - echo -e "*** $(date -u) Getting a non privileged user\n" |& tee -a clientTestOutputs.txt dirac-login -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt +#-------------------------------------------------------------------------------# +echo -e "*** $(date -u) **** Submit a job ****\n" + echo -e '[\n Arguments = "Hello World";\n Executable = "echo";\n Site = "DIRAC.Jenkins.ch";' > test.jdl echo " JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test.jdl echo "]" >> test.jdl -dirac-wms-job-submit test.jdl +dirac-wms-job-submit test.jdl |& tee -a clientTestOutputs.txt + +#-------------------------------------------------------------------------------# +echo -e "*** $(date -u) **** add a file ****\n" + +base64 /dev/urandom | head -c 10000 > test_lfn.txt +dirac-dms-add-file LFN:/vo/test_lfn.txt test_lfn.txt S3-DIRECT |& tee -a clientTestOutputs.txt + +#-------------------------------------------------------------------------------# +echo -e "*** $(date -u) **** Submit a job with an input ****\n" + +echo -e '[\n Arguments = "Hello World";\n Executable = "echo";\n Site = "DIRAC.Jenkins.ch";' > test_dl.jdl +echo " InputData = \"/vo/test_lfn.txt\";" >> test_dl.jdl; +echo " JobName = \"${GITHUB_JOB}_$(date +"%Y-%m-%d_%T" | sed 's/://g')\"" >> test_dl.jdl +echo "]" >> test_dl.jdl +dirac-wms-job-submit test_dl.jdl |& tee -a clientTestOutputs.txt + diff --git a/tests/Jenkins/dirac-cfg-update-server.py b/tests/Jenkins/dirac-cfg-update-server.py index 97824e06ea1..57eefad4947 100644 --- a/tests/Jenkins/dirac-cfg-update-server.py +++ b/tests/Jenkins/dirac-cfg-update-server.py @@ -127,6 +127,7 @@ # DIRAC.Jenkins.ch # { # SE = SE-1 +# SE += S3-DIRECT # CEs # { # jenkins.cern.ch @@ -163,7 +164,7 @@ print(res["Message"]) sys.exit(1) -csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/SE", "SE-1") +csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/SE", "SE-1,S3-DIRECT") csAPI.setOption("Resources/Sites/DIRAC/DIRAC.Jenkins.ch/CEs/jenkins.cern.ch/CEType", "Test") csAPI.setOption( "Resources/Sites/DIRAC/DIRAC.Jenkins.ch/CEs/jenkins.cern.ch/Queues/jenkins-queue_not_important/maxCPUTime", "200000" diff --git a/tests/Workflow/Integration/Test_UserJobs.py b/tests/Workflow/Integration/Test_UserJobs.py index 29cbc6663a4..e7f64450eb3 100644 --- a/tests/Workflow/Integration/Test_UserJobs.py +++ b/tests/Workflow/Integration/Test_UserJobs.py @@ -3,22 +3,21 @@ Can be automatized. """ # pylint: disable=protected-access, wrong-import-position, invalid-name, missing-docstring +import multiprocessing import os import sys import unittest -import multiprocessing import DIRAC DIRAC.initialize(extra_config_files=["pilot.cfg"]) # Initialize configuration from DIRAC import gLogger, rootPath +from DIRAC.Interfaces.API.Dirac import Dirac +from DIRAC.Interfaces.API.Job import Job from DIRAC.tests.Utilities.IntegrationTest import IntegrationTest from DIRAC.tests.Utilities.utils import find_all -from DIRAC.Interfaces.API.Job import Job -from DIRAC.Interfaces.API.Dirac import Dirac - class UserJobTestCase(IntegrationTest): """Base class for the UserJob test cases""" @@ -154,6 +153,27 @@ def test_execute(self): self.assertTrue(res["OK"]) +class DownloadSuccess(UserJobTestCase): + def test_execute(self): + """Testing a job that downloads inputs""" + + job = Job() + + job.setName("dl-test") + job.setExecutable("/bin/ls", "-l") + job.setLogLevel("DEBUG") + try: + # This is the standard location in Jenkins + job.setInputSandbox(find_all("pilot.cfg", os.environ["WORKSPACE"] + "/PilotInstallDIR")[0]) + except (IndexError, KeyError): + job.setInputSandbox(find_all("pilot.cfg", rootPath)[0]) + job.setConfigArgs("pilot.cfg") + job.setInputData("/vo/test_lfn.txt") + job.setInputDataPolicy("Download") + res = job.runLocal(self.d) + self.assertTrue(res["OK"]) + + class MPSuccess(UserJobTestCase): def test_fixed(self): """this tests executes a job that requires exactly 4 processors""" @@ -245,6 +265,7 @@ def test_min1(self): suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(HelloWorldSuccess)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(HelloWorldPlusSuccess)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(LSSuccess)) + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(DownloadSuccess)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(MPSuccess)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(MPSuccessMinMax)) testResult = unittest.TextTestRunner(verbosity=2).run(suite)