From de918698203ec8287e783ee43a92405fb66ece24 Mon Sep 17 00:00:00 2001 From: dg1223 Date: Sat, 30 Mar 2024 13:47:47 -0400 Subject: [PATCH 1/9] added default values to init function (#2309) --- ganga/GangaCore/Lib/Virtualization/Docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ganga/GangaCore/Lib/Virtualization/Docker.py b/ganga/GangaCore/Lib/Virtualization/Docker.py index e5411199d7..1e2ee73960 100644 --- a/ganga/GangaCore/Lib/Virtualization/Docker.py +++ b/ganga/GangaCore/Lib/Virtualization/Docker.py @@ -36,7 +36,7 @@ class Docker(IVirtualization): _schema = IVirtualization._schema.inherit_copy() _schema.datadict['mode'] = SimpleItem(defvalue="P1", doc='Mode of container execution') - def __init__(self, image, mode): + def __init__(self, image='', mode='P1'): super().__init__(image) self.mode = mode From f3c640842b5ef2ca7a69435291ab70ccf5fca27e Mon Sep 17 00:00:00 2001 From: dg1223 Date: Sat, 30 Mar 2024 13:48:02 -0400 Subject: [PATCH 2/9] test: added new unit tests to check Docker initialization (#2309) --- .../TestVirtualizationDocker.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 ganga/GangaCore/test/Unit/Virtualization/TestVirtualizationDocker.py diff --git a/ganga/GangaCore/test/Unit/Virtualization/TestVirtualizationDocker.py b/ganga/GangaCore/test/Unit/Virtualization/TestVirtualizationDocker.py new file mode 100644 index 0000000000..970f242d5e --- /dev/null +++ b/ganga/GangaCore/test/Unit/Virtualization/TestVirtualizationDocker.py @@ -0,0 +1,84 @@ +from GangaCore.testlib.GangaUnitTest import GangaUnitTest +from GangaCore.Lib.Virtualization import Docker +from GangaTest.Framework.utils import sleep_until_completed, file_contains + +class TestDocker(GangaUnitTest): + def test_DockerNoImageArg(self): + """ + The test_DockerNoImageArg function tests that Ganga can + submit a virtualization job using the default Docker image. + + This test will fail if Ganga cannot find the job in the registry or + it cannot intialize the container with the default Docker image. + """ + from GangaCore.GPI import Job + j = Job(name="dockertest") + self.assertEqual(j.name, 'dockertest', "Something wrong with how\ +Job was imported or initialized.") + j.virtualization = Docker() + self.assertEqual(j.virtualization.image, '', "Docker image was not\ +passed correctly as an argument.") + j.submit() + sleep_until_completed(j) + j.remove() + + def test_DockerBareImageArg(self): + """ + The test_DockerBareImageArg function tests that the Docker image + is passed correctly as an argument without the 'image=' prefix. + + This test will fail if Ganga cannot find the job in the registry or + it cannot intialize the container with the desired Docker image. + """ + from GangaCore.GPI import Job + j = Job(name="dockertest") + self.assertEqual(j.name, 'dockertest', "Something wrong with how\ +Job was imported or initialized.") + j.virtualization = Docker('fedora:latest') + self.assertEqual(j.virtualization.image, 'fedora:latest', "Docker\ +image was not passed correctly as an argument.") + j.submit() + sleep_until_completed(j) + j.remove() + + def test_DockerImageArg(self): + """ + The DockerImageArg function tests that the Docker image + is passed correctly as an argument with the 'image=' prefix. + + This test will fail if Ganga cannot find the job in the registry or + it cannot intialize the container with the desired Docker image. + """ + from GangaCore.GPI import Job + j = Job(name="dockertest") + self.assertEqual(j.name, 'dockertest', "Something wrong with how\ +Job was imported or initialized.") + j.virtualization = Docker(image='fedora:latest') + self.assertEqual(j.virtualization.image, 'fedora:latest', "Docker\ +image was not passed correctly as an argument.") + j.submit() + sleep_until_completed(j) + j.remove() + + def test_DockerImageArgWithMode(self): + """ + The DockerImageArgWithMode function tests that the Docker image + is passed correctly as an argument with the 'image=' prefix. It + also test if mode is explicitly set to 'P1'. + + This test will fail if Ganga cannot find the job in the registry + or it cannot intialize the container with the desired Docker image + and mode. + """ + from GangaCore.GPI import Job + j = Job(name="dockertest") + self.assertEqual(j.name, 'dockertest', "Something wrong with how\ +Job was imported or initialized.") + j.virtualization = Docker(image='fedora:latest', mode='P1') + self.assertEqual(j.virtualization.image, 'fedora:latest', "Docker\ +image was not passed correctly as an argument.") + self.assertEqual(j.virtualization.mode, 'P1', "Mode was not passed\ +correctly as an argument.") + j.submit() + sleep_until_completed(j) + j.remove() From 62c692518b6a01208decab6d2381fe3408e53aaa Mon Sep 17 00:00:00 2001 From: dg1223 Date: Sat, 30 Mar 2024 13:53:29 -0400 Subject: [PATCH 3/9] fixed lint issues (#2309) --- .../Virtualization/TestVirtualizationDocker.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ganga/GangaCore/test/Unit/Virtualization/TestVirtualizationDocker.py b/ganga/GangaCore/test/Unit/Virtualization/TestVirtualizationDocker.py index 970f242d5e..bcf7af9954 100644 --- a/ganga/GangaCore/test/Unit/Virtualization/TestVirtualizationDocker.py +++ b/ganga/GangaCore/test/Unit/Virtualization/TestVirtualizationDocker.py @@ -1,13 +1,14 @@ from GangaCore.testlib.GangaUnitTest import GangaUnitTest from GangaCore.Lib.Virtualization import Docker -from GangaTest.Framework.utils import sleep_until_completed, file_contains +from GangaTest.Framework.utils import sleep_until_completed + class TestDocker(GangaUnitTest): def test_DockerNoImageArg(self): """ - The test_DockerNoImageArg function tests that Ganga can + The test_DockerNoImageArg function tests that Ganga can submit a virtualization job using the default Docker image. - + This test will fail if Ganga cannot find the job in the registry or it cannot intialize the container with the default Docker image. """ @@ -21,12 +22,12 @@ def test_DockerNoImageArg(self): j.submit() sleep_until_completed(j) j.remove() - + def test_DockerBareImageArg(self): """ The test_DockerBareImageArg function tests that the Docker image is passed correctly as an argument without the 'image=' prefix. - + This test will fail if Ganga cannot find the job in the registry or it cannot intialize the container with the desired Docker image. """ @@ -45,7 +46,7 @@ def test_DockerImageArg(self): """ The DockerImageArg function tests that the Docker image is passed correctly as an argument with the 'image=' prefix. - + This test will fail if Ganga cannot find the job in the registry or it cannot intialize the container with the desired Docker image. """ @@ -59,13 +60,13 @@ def test_DockerImageArg(self): j.submit() sleep_until_completed(j) j.remove() - + def test_DockerImageArgWithMode(self): """ The DockerImageArgWithMode function tests that the Docker image is passed correctly as an argument with the 'image=' prefix. It also test if mode is explicitly set to 'P1'. - + This test will fail if Ganga cannot find the job in the registry or it cannot intialize the container with the desired Docker image and mode. From 2c7a60fda39bda1a4002e04c2176b0da77b48359 Mon Sep 17 00:00:00 2001 From: Mark Smith Date: Fri, 12 Apr 2024 09:23:50 +0200 Subject: [PATCH 4/9] add apptainer to build --- ganga/GangaLHCb/Lib/Applications/GaudiExec.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ganga/GangaLHCb/Lib/Applications/GaudiExec.py b/ganga/GangaLHCb/Lib/Applications/GaudiExec.py index d00c5d54bd..a4fb43ab1d 100755 --- a/ganga/GangaLHCb/Lib/Applications/GaudiExec.py +++ b/ganga/GangaLHCb/Lib/Applications/GaudiExec.py @@ -536,6 +536,8 @@ def execCmd(self, cmd): cmd_file.write("#!/bin/bash") cmd_file.write("\n") +# if self.getJobObject().virtualization: +# cmd_file.write('apptainer exec --env "PATH=$PATH" --bind $PWD --bind /cvmfs:/cvmfs:ro /cvmfs/cernvm-prod.cern.ch/cvm4 ') cmd_file.write(self.getEnvScript(isLbEnv)) cmd_file.write(cmd) cmd_file.flush() @@ -564,7 +566,9 @@ def execCmd(self, cmd): if cmd != 'make': rc, stdout, stderr = _exec_cmd(cmd_file.name, self.directory) else: - rc, stdout, stderr = _exec_cmd(cmd_file.name, self.directory) + cmd_to_run = 'apptainer exec --env "PATH=$PATH" --bind $PWD --bind /cvmfs:/cvmfs:ro /cvmfs/cernvm-prod.cern.ch/cvm4 ' + cmd_file.name + rc, stdout, stderr = _exec_cmd(cmd_to_run, self.directory) +# rc, stdout, stderr = _exec_cmd(cmd_file.name, self.directory) if rc != 0: logger.error("Failed to execute command: %s" % cmd_file.name) logger.error("Tried to execute command in: %s" % self.directory) From 284342f88cb57d7a28b56e2e9c22636f8433d799 Mon Sep 17 00:00:00 2001 From: Mark Smith Date: Fri, 12 Apr 2024 09:36:30 +0200 Subject: [PATCH 5/9] some fix --- ganga/GangaLHCb/Lib/RTHandlers/GaudiExecRTHandlers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ganga/GangaLHCb/Lib/RTHandlers/GaudiExecRTHandlers.py b/ganga/GangaLHCb/Lib/RTHandlers/GaudiExecRTHandlers.py index 1820706a3e..04a0bfad1c 100755 --- a/ganga/GangaLHCb/Lib/RTHandlers/GaudiExecRTHandlers.py +++ b/ganga/GangaLHCb/Lib/RTHandlers/GaudiExecRTHandlers.py @@ -794,6 +794,9 @@ def flush_streams(pipe): print("Executing: %s" % '###COMMAND###'+' '+' '.join(sys.argv[1:])) + ###VIRTUALIZATION### + + # Execute the actual command on the WN # NB os.system caused the entire stream to be captured before being streamed in some cases pipe = subprocess.Popen('###COMMAND###'+' '+' '.join(sys.argv[1:]), shell=True, From 5dd0d1672350e7b327a317bac4941e9e5ff83b6f Mon Sep 17 00:00:00 2001 From: Mark Smith Date: Fri, 12 Apr 2024 09:37:47 +0200 Subject: [PATCH 6/9] tidy --- ganga/GangaLHCb/Lib/RTHandlers/GaudiExecRTHandlers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ganga/GangaLHCb/Lib/RTHandlers/GaudiExecRTHandlers.py b/ganga/GangaLHCb/Lib/RTHandlers/GaudiExecRTHandlers.py index 04a0bfad1c..1820706a3e 100755 --- a/ganga/GangaLHCb/Lib/RTHandlers/GaudiExecRTHandlers.py +++ b/ganga/GangaLHCb/Lib/RTHandlers/GaudiExecRTHandlers.py @@ -794,9 +794,6 @@ def flush_streams(pipe): print("Executing: %s" % '###COMMAND###'+' '+' '.join(sys.argv[1:])) - ###VIRTUALIZATION### - - # Execute the actual command on the WN # NB os.system caused the entire stream to be captured before being streamed in some cases pipe = subprocess.Popen('###COMMAND###'+' '+' '.join(sys.argv[1:]), shell=True, From 41cd785d9562e3e7f16d98bc8299bd171955e485 Mon Sep 17 00:00:00 2001 From: Mark Smith Date: Fri, 12 Apr 2024 09:41:06 +0200 Subject: [PATCH 7/9] tidy --- ganga/GangaLHCb/Lib/Applications/GaudiExec.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ganga/GangaLHCb/Lib/Applications/GaudiExec.py b/ganga/GangaLHCb/Lib/Applications/GaudiExec.py index a4fb43ab1d..9b92f79516 100755 --- a/ganga/GangaLHCb/Lib/Applications/GaudiExec.py +++ b/ganga/GangaLHCb/Lib/Applications/GaudiExec.py @@ -536,8 +536,6 @@ def execCmd(self, cmd): cmd_file.write("#!/bin/bash") cmd_file.write("\n") -# if self.getJobObject().virtualization: -# cmd_file.write('apptainer exec --env "PATH=$PATH" --bind $PWD --bind /cvmfs:/cvmfs:ro /cvmfs/cernvm-prod.cern.ch/cvm4 ') cmd_file.write(self.getEnvScript(isLbEnv)) cmd_file.write(cmd) cmd_file.flush() From 547f514b2aee3043c3df0cee6fa046f5834fa9c5 Mon Sep 17 00:00:00 2001 From: Mark Smith Date: Fri, 12 Apr 2024 09:44:33 +0200 Subject: [PATCH 8/9] add option --- ganga/GangaLHCb/Lib/Applications/GaudiExec.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ganga/GangaLHCb/Lib/Applications/GaudiExec.py b/ganga/GangaLHCb/Lib/Applications/GaudiExec.py index 9b92f79516..6826f82dff 100755 --- a/ganga/GangaLHCb/Lib/Applications/GaudiExec.py +++ b/ganga/GangaLHCb/Lib/Applications/GaudiExec.py @@ -195,7 +195,7 @@ class GaudiExec(IPrepareApp): 'nMakeCores': SimpleItem(defvalue=1, doc='Number of cores to be provided via the "-j" option to the "make" command'\ 'when building the ganga-input-sandbox'), - + 'apptainerBuild' : SimpleItem(defvaule=False, doc="Run the build command in apptainer"), # Prepared job object 'is_prepared': SimpleItem(defvalue=None, strict_sequence=0, visitable=1, copyable=1, hidden=0, typelist=[None, ShareDir], protected=0, comparable=1, @@ -564,9 +564,11 @@ def execCmd(self, cmd): if cmd != 'make': rc, stdout, stderr = _exec_cmd(cmd_file.name, self.directory) else: - cmd_to_run = 'apptainer exec --env "PATH=$PATH" --bind $PWD --bind /cvmfs:/cvmfs:ro /cvmfs/cernvm-prod.cern.ch/cvm4 ' + cmd_file.name - rc, stdout, stderr = _exec_cmd(cmd_to_run, self.directory) -# rc, stdout, stderr = _exec_cmd(cmd_file.name, self.directory) + if self.apptainerBuild or 'slc6' in self.platform: + cmd_to_run = 'apptainer exec --env "PATH=$PATH" --bind $PWD --bind /cvmfs:/cvmfs:ro /cvmfs/cernvm-prod.cern.ch/cvm4 ' + cmd_file.name + rc, stdout, stderr = _exec_cmd(cmd_to_run, self.directory) + else: + rc, stdout, stderr = _exec_cmd(cmd_file.name, self.directory) if rc != 0: logger.error("Failed to execute command: %s" % cmd_file.name) logger.error("Tried to execute command in: %s" % self.directory) From 6b0441cdf691f611b18ac8e6b8c10e48a5b5b06c Mon Sep 17 00:00:00 2001 From: Mark Smith Date: Fri, 12 Apr 2024 10:32:24 +0200 Subject: [PATCH 9/9] add options --- ganga/GangaLHCb/Lib/Applications/GaudiExec.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ganga/GangaLHCb/Lib/Applications/GaudiExec.py b/ganga/GangaLHCb/Lib/Applications/GaudiExec.py index 6826f82dff..b453dcd9df 100755 --- a/ganga/GangaLHCb/Lib/Applications/GaudiExec.py +++ b/ganga/GangaLHCb/Lib/Applications/GaudiExec.py @@ -195,7 +195,9 @@ class GaudiExec(IPrepareApp): 'nMakeCores': SimpleItem(defvalue=1, doc='Number of cores to be provided via the "-j" option to the "make" command'\ 'when building the ganga-input-sandbox'), - 'apptainerBuild' : SimpleItem(defvaule=False, doc="Run the build command in apptainer"), + 'apptainerBuild' : SimpleItem(defvalue=False, doc="Run the build command in apptainer"), + 'containerLocation' : SimpleItem(defvalue='/cvmfs/cernvm-prod.cern.ch/cvm4', + doc='Where is the container to use for the build located'), # Prepared job object 'is_prepared': SimpleItem(defvalue=None, strict_sequence=0, visitable=1, copyable=1, hidden=0, typelist=[None, ShareDir], protected=0, comparable=1, @@ -565,8 +567,15 @@ def execCmd(self, cmd): rc, stdout, stderr = _exec_cmd(cmd_file.name, self.directory) else: if self.apptainerBuild or 'slc6' in self.platform: - cmd_to_run = 'apptainer exec --env "PATH=$PATH" --bind $PWD --bind /cvmfs:/cvmfs:ro /cvmfs/cernvm-prod.cern.ch/cvm4 ' + cmd_file.name - rc, stdout, stderr = _exec_cmd(cmd_to_run, self.directory) + try: + logger.info('Building inside apptainer: %s' % self.containerLocation) + cmd_to_run = 'apptainer exec --env "PATH=$PATH" --bind $PWD --bind /cvmfs:/cvmfs:ro '\ + + self.containerLocation + ' ' + cmd_file.name + rc, stdout, stderr = _exec_cmd(cmd_to_run, self.directory) + except: + logger.error('Failed to build the application inside a container. ' + 'Perhaps the specified container location is not accessible.') + raise GangaException('Failed to execute make command') else: rc, stdout, stderr = _exec_cmd(cmd_file.name, self.directory) if rc != 0: