From 251a2e0396ca9dcd6921d9a5f6f663c6daf91eeb Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 26 Sep 2023 17:58:20 +0200 Subject: [PATCH 1/8] test: added an integration test --- .github/workflows/integration.yml | 46 +++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/integration.yml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 00000000..432e136e --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,46 @@ +name: integration + +on: [push, pull_request] + +jobs: + integration: + runs-on: ubuntu-latest + + strategy: + matrix: + python: + - 2.7.18 + - 3.6.15 + - 3.9.17 + dirac_version: + - rel-v8r0 + - integration + + container: python:${{ matrix.python }}-slim + + steps: + - uses: actions/checkout@v4 + - name: Retrieve the secret and decode it to a file + env: + HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} + HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + run: | + cd Pilot + mkdir -p etc/grid-security + echo $HOSTCERT_BASE64 | base64 --decode > etc/grid-security/hostcert.pem + echo $HOSTKEY_BASE64 | base64 --decode > etc/grid-security/hostkey.pem + chmod 440 etc/grid-security/hostcert.pem + chmod 400 etc/grid-security/hostkey.pem + - name: tests + run: | + cd Pilot + cp ../tests/CI/pilot_oldSchema.json pilot.json + sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json + sed -i "s/VAR_JENKINS_CE/jenkins.cern.ch/g" pilot.json + sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json + sed -i "s/VAR_DIRAC_VERSION/${{ matrix.dirac_version }}/g" pilot.json + sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json + sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json + sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + cat pilot.cfg From ef7f9aaa6f00f8aa960dd1fa8c92d8309faebebd Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 3 Oct 2023 11:51:31 +0200 Subject: [PATCH 2/8] test: added the matching test --- .github/workflows/integration.yml | 86 +++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 432e136e..de6dcbe2 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -44,3 +44,89 @@ jobs: sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug cat pilot.cfg + + matching: + runs-on: ubuntu-latest + + container: python:3.9.17-slim + + + steps: + - uses: actions/checkout@v4 + - name: Retrieve the secret and decode it to a file + env: + HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} + HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + run: | + cd Pilot + mkdir -p etc/grid-security + echo $HOSTCERT_BASE64 | base64 --decode > etc/grid-security/hostcert.pem + echo $HOSTKEY_BASE64 | base64 --decode > etc/grid-security/hostkey.pem + chmod 440 etc/grid-security/hostcert.pem + chmod 400 etc/grid-security/hostkey.pem + - name: tests + run: | + cd Pilot + cp ../tests/CI/pilot_oldSchema.json pilot.json + sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json + sed -i "s/VAR_JENKINS_CE/jenkins-full.cern.ch/g" pilot.json + sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json + sed -i "s/VAR_DIRAC_VERSION/integration/g" pilot.json + sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json + sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json + sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + + + ################################## + ### # extensions tests + ################################## + + ################################## + #### LHCb + + ext-lhcb_integration: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: add cvmfs + run: | + wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest_all.deb + sudo dpkg -i cvmfs-release-latest_all.deb + rm -f cvmfs-release-latest_all.deb + sudo apt-get update + sudo apt-get install cvmfs + sudo cvmfs_config setup + - name: Retrieve the secret and decode it to a file + env: + HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} + HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + run: | + cd Pilot + mkdir -p etc/grid-security/vomses + mkdir -p etc/grid-security/vomsdir + mkdir -p etc/grid-security/certificates + echo $HOSTCERT_BASE64 | base64 --decode > etc/grid-security/hostcert.pem + echo $HOSTKEY_BASE64 | base64 --decode > etc/grid-security/hostkey.pem + chmod 440 etc/grid-security/hostcert.pem + chmod 400 etc/grid-security/hostkey.pem + - name: tests + run: | + cd Pilot + export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot + echo $VO_LHCB_SW_DIR + curl https://lhcbdirac.s3.cern.ch/Pilot3/LHCbPilotCommands.py -o LHCbPilotCommands.py + cp ../tests/CI/pilot_oldSchema.json pilot.json + sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json + sed -i "s/VAR_JENKINS_CE/jenkins-lhcb.cern.ch/g" pilot.json + sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json + sed -i "s/VAR_DIRAC_VERSION/master/g" pilot.json + sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json + sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json + sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + ls -al + python dirac-pilot.py --modules https://gitlab.cern.ch/lhcb-dirac/LHCbDIRAC.git:::LHCbDIRAC:::master -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + cat pilot.cfg + \ No newline at end of file From 118f06dbb184594b87597a9c531d4fac28e21fdc Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 3 Oct 2023 15:51:39 +0200 Subject: [PATCH 3/8] test: added LHCbPilot integration test F --- .github/workflows/integration.yml | 47 ++++++++++++++++--------------- tests/CI/pilot_newSchema.json | 36 +++++++++++++++++++++-- tests/CI/pilot_oldSchema.json | 16 +++++++++++ 3 files changed, 74 insertions(+), 25 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index de6dcbe2..2849cb39 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,6 +1,6 @@ name: integration -on: [push, pull_request] +on: [push] jobs: integration: @@ -27,10 +27,11 @@ jobs: run: | cd Pilot mkdir -p etc/grid-security - echo $HOSTCERT_BASE64 | base64 --decode > etc/grid-security/hostcert.pem - echo $HOSTKEY_BASE64 | base64 --decode > etc/grid-security/hostkey.pem + echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem + echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem chmod 440 etc/grid-security/hostcert.pem chmod 400 etc/grid-security/hostkey.pem + ls -l etc/grid-security - name: tests run: | cd Pilot @@ -45,12 +46,19 @@ jobs: python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug cat pilot.cfg + matching: runs-on: ubuntu-latest - container: python:3.9.17-slim - + strategy: + matrix: + python: + - 2.7.18 + - 3.6.15 + - 3.9.17 + container: python:${{ matrix.python }}-slim + steps: - uses: actions/checkout@v4 - name: Retrieve the secret and decode it to a file @@ -60,8 +68,8 @@ jobs: run: | cd Pilot mkdir -p etc/grid-security - echo $HOSTCERT_BASE64 | base64 --decode > etc/grid-security/hostcert.pem - echo $HOSTKEY_BASE64 | base64 --decode > etc/grid-security/hostkey.pem + echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem + echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem chmod 440 etc/grid-security/hostcert.pem chmod 400 etc/grid-security/hostkey.pem - name: tests @@ -90,15 +98,11 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: cvmfs-contrib/github-action-cvmfs@v3 + + - name: Test CernVM-FS + run: ls /cvmfs/lhcb.cern.ch - - name: add cvmfs - run: | - wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest_all.deb - sudo dpkg -i cvmfs-release-latest_all.deb - rm -f cvmfs-release-latest_all.deb - sudo apt-get update - sudo apt-get install cvmfs - sudo cvmfs_config setup - name: Retrieve the secret and decode it to a file env: HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} @@ -108,25 +112,22 @@ jobs: mkdir -p etc/grid-security/vomses mkdir -p etc/grid-security/vomsdir mkdir -p etc/grid-security/certificates - echo $HOSTCERT_BASE64 | base64 --decode > etc/grid-security/hostcert.pem - echo $HOSTKEY_BASE64 | base64 --decode > etc/grid-security/hostkey.pem + echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem + echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem chmod 440 etc/grid-security/hostcert.pem chmod 400 etc/grid-security/hostkey.pem - name: tests run: | cd Pilot export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot - echo $VO_LHCB_SW_DIR curl https://lhcbdirac.s3.cern.ch/Pilot3/LHCbPilotCommands.py -o LHCbPilotCommands.py cp ../tests/CI/pilot_oldSchema.json pilot.json sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json sed -i "s/VAR_JENKINS_CE/jenkins-lhcb.cern.ch/g" pilot.json sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json - sed -i "s/VAR_DIRAC_VERSION/master/g" pilot.json + sed -i "s/VAR_DIRAC_VERSION//g" pilot.json sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - ls -al - python dirac-pilot.py --modules https://gitlab.cern.ch/lhcb-dirac/LHCbDIRAC.git:::LHCbDIRAC:::master -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug - cat pilot.cfg - \ No newline at end of file + echo $SHELL + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index e81113b9..1d57c9f7 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -69,7 +69,15 @@ "VAR_JENKINS_QUEUE": { "LocalCEType": "Pool/Singularity" } - } + }, + "jenkins-lhcb.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCb" + }, + "jenkins-lhcb-d.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCbD" + } }, "GenericPilotDNs": [], "Defaults": { @@ -127,8 +135,32 @@ "ConfigureArchitecture", "ConfigureCPURequirements", "LaunchAgent" + ], + "LHCb": [ + "CheckWorkerNode", + "LHCbInstallDIRAC", + "LHCbConfigureBasics", + "RegisterPilot", + "CheckCECapabilities", + "LHCbAddCVMFSTags", + "CheckWNCapabilities", + "LHCbConfigureSite", + "LHCbConfigureArchitecture", + "LHCbConfigureCPURequirements" + ], + "LHCbD": [ + "CheckWorkerNode", + "InstallDIRAC", + "LHCbConfigureBasics", + "RegisterPilot", + "CheckCECapabilities", + "LHCbAddCVMFSTags", + "CheckWNCapabilities", + "LHCbConfigureSite", + "LHCbConfigureArchitecture", + "LHCbConfigureCPURequirements" ] - } + } } }, "gridpp": { diff --git a/tests/CI/pilot_oldSchema.json b/tests/CI/pilot_oldSchema.json index f4be1881..a6aa2bdc 100644 --- a/tests/CI/pilot_oldSchema.json +++ b/tests/CI/pilot_oldSchema.json @@ -36,6 +36,18 @@ "ConfigureArchitecture", "ConfigureCPURequirements", "LaunchAgent" + ], + "LHCb": [ + "CheckWorkerNode", + "LHCbInstallDIRAC", + "LHCbConfigureBasics", + "RegisterPilot", + "CheckCECapabilities", + "LHCbAddCVMFSTags", + "CheckWNCapabilities", + "LHCbConfigureSite", + "LHCbConfigureArchitecture", + "LHCbConfigureCPURequirements" ] }, "Logging": { @@ -146,6 +158,10 @@ "VAR_JENKINS_QUEUE": { "LocalCEType": "Pool/Singularity" } + }, + "jenkins-lhcb.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCb" } }, "DefaultSetup": "DIRAC-Certification" From 1442bb319d34f6f455c07325413b436308666351 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 5 Oct 2023 12:02:49 +0200 Subject: [PATCH 4/8] test: added integration test using CVMFS --- .github/workflows/integration.yml | 41 +++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 2849cb39..c85e6525 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -3,7 +3,7 @@ name: integration on: [push] jobs: - integration: + integration-local_install: runs-on: ubuntu-latest strategy: @@ -47,6 +47,44 @@ jobs: cat pilot.cfg + integration-cvmfs: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: cvmfs-contrib/github-action-cvmfs@v3 + + - name: Test CernVM-FS + run: ls /cvmfs/dirac.egi.eu + + - name: Retrieve the secret and decode it to a file + env: + HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} + HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + run: | + cd Pilot + mkdir -p etc/grid-security + echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem + echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem + chmod 440 etc/grid-security/hostcert.pem + chmod 400 etc/grid-security/hostkey.pem + ls -l etc/grid-security + - name: tests + run: | + cd Pilot + cp ../tests/CI/pilot_oldSchema.json pilot.json + sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json + sed -i "s/VAR_JENKINS_CE/jenkins.cern.ch/g" pilot.json + sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json + sed -i "s/VAR_DIRAC_VERSION/v8.1.0a16/g" pilot.json + sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json + sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json + sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + cat pilot.cfg + + + matching: runs-on: ubuntu-latest @@ -129,5 +167,4 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - echo $SHELL python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug From b8dafd1c9470bd3bbcd9c77b9556b945468006d3 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 5 Oct 2023 12:46:16 +0200 Subject: [PATCH 5/8] fix: added few debug messages --- .github/workflows/integration.yml | 2 +- Pilot/pilotCommands.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c85e6525..c4060893 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -163,7 +163,7 @@ jobs: sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json sed -i "s/VAR_JENKINS_CE/jenkins-lhcb.cern.ch/g" pilot.json sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json - sed -i "s/VAR_DIRAC_VERSION//g" pilot.json + sed -i "s/VAR_DIRAC_VERSION/v11.0.20/g" pilot.json sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 16779ace..998f82dc 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -302,12 +302,17 @@ def _locateInstallationScript(self): def _getPreinstalledEnvScript(self): """ Get preinstalled environment script if any """ + self.log.debug("self.pp.preinstalledEnv = %s" % self.pp.preinstalledEnv) + self.log.debug("self.pp.preinstalledEnvPrefix = %s" % self.pp.preinstalledEnvPrefix) + preinstalledEnvScript = self.pp.preinstalledEnv if not preinstalledEnvScript and self.pp.preinstalledEnvPrefix: version = self.pp.releaseVersion or "pro" arch = platform.system() + "-" + platform.machine() preinstalledEnvScript = os.path.join(self.pp.preinstalledEnvPrefix, version, arch, "diracosrc") + self.log.debug("preinstalledEnvScript = %s" % preinstalledEnvScript) + if preinstalledEnvScript: self.log.info("Evaluating env script %s" % preinstalledEnvScript) if not safe_listdir(os.path.dirname(preinstalledEnvScript)): From 2d4ebd419ba824d3d6e66306e809715830d7be91 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 5 Oct 2023 14:05:42 +0200 Subject: [PATCH 6/8] test: added an LHCb integration test using vanilla Pilot install command --- .github/workflows/integration.yml | 38 +++++++++++++++++++++++++++++++ tests/CI/pilot_newSchema.json | 2 +- tests/CI/pilot_oldSchema.json | 16 +++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c4060893..429af736 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -168,3 +168,41 @@ jobs: sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + + ext-lhcb_integration_dirac_installer: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: cvmfs-contrib/github-action-cvmfs@v3 + + - name: Test CernVM-FS + run: ls /cvmfs/lhcb.cern.ch + + - name: Retrieve the secret and decode it to a file + env: + HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} + HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + run: | + cd Pilot + mkdir -p etc/grid-security/vomses + mkdir -p etc/grid-security/vomsdir + mkdir -p etc/grid-security/certificates + echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem + echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem + chmod 440 etc/grid-security/hostcert.pem + chmod 400 etc/grid-security/hostkey.pem + - name: tests + run: | + cd Pilot + export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot + curl https://lhcbdirac.s3.cern.ch/Pilot3/LHCbPilotCommands.py -o LHCbPilotCommands.py + cp ../tests/CI/pilot_oldSchema.json pilot.json + sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json + sed -i "s/VAR_JENKINS_CE/jenkins-lhcb.cern.ch/g" pilot.json + sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json + sed -i "s/VAR_DIRAC_VERSION/v11.0.20/g" pilot.json + sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json + sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json + sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb_d.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug diff --git a/tests/CI/pilot_newSchema.json b/tests/CI/pilot_newSchema.json index 1d57c9f7..002deffc 100644 --- a/tests/CI/pilot_newSchema.json +++ b/tests/CI/pilot_newSchema.json @@ -77,7 +77,7 @@ "jenkins-lhcb-d.cern.ch": { "Site": "VAR_JENKINS_SITE", "GridCEType": "LHCbD" - } + } }, "GenericPilotDNs": [], "Defaults": { diff --git a/tests/CI/pilot_oldSchema.json b/tests/CI/pilot_oldSchema.json index a6aa2bdc..3bdadbfb 100644 --- a/tests/CI/pilot_oldSchema.json +++ b/tests/CI/pilot_oldSchema.json @@ -48,6 +48,18 @@ "LHCbConfigureSite", "LHCbConfigureArchitecture", "LHCbConfigureCPURequirements" + ], + "LHCbD": [ + "CheckWorkerNode", + "InstallDIRAC", + "LHCbConfigureBasics", + "RegisterPilot", + "CheckCECapabilities", + "LHCbAddCVMFSTags", + "CheckWNCapabilities", + "LHCbConfigureSite", + "LHCbConfigureArchitecture", + "LHCbConfigureCPURequirements" ] }, "Logging": { @@ -162,6 +174,10 @@ "jenkins-lhcb.cern.ch": { "Site": "VAR_JENKINS_SITE", "GridCEType": "LHCb" + }, + "jenkins-lhcb-d.cern.ch": { + "Site": "VAR_JENKINS_SITE", + "GridCEType": "LHCbD" } }, "DefaultSetup": "DIRAC-Certification" From b244b07bad35a49a3d32008cd336d7cb2fe3505e Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 6 Oct 2023 10:46:44 +0200 Subject: [PATCH 7/8] fix: added security locations config --- .github/workflows/integration.yml | 17 +++- Pilot/pilotCommands.py | 126 ++++++++++++++++++++++++------ Pilot/pilotTools.py | 4 + 3 files changed, 120 insertions(+), 27 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 429af736..a1cc31b4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,7 +31,6 @@ jobs: echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem chmod 440 etc/grid-security/hostcert.pem chmod 400 etc/grid-security/hostkey.pem - ls -l etc/grid-security - name: tests run: | cd Pilot @@ -68,8 +67,11 @@ jobs: echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem chmod 440 etc/grid-security/hostcert.pem chmod 400 etc/grid-security/hostkey.pem - ls -l etc/grid-security - name: tests + env: + X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses run: | cd Pilot cp ../tests/CI/pilot_oldSchema.json pilot.json @@ -84,7 +86,6 @@ jobs: cat pilot.cfg - matching: runs-on: ubuntu-latest @@ -155,6 +156,10 @@ jobs: chmod 440 etc/grid-security/hostcert.pem chmod 400 etc/grid-security/hostkey.pem - name: tests + env: + X509_CERT_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/lhcb.cern.ch/etc/grid-security/vomses run: | cd Pilot export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot @@ -193,6 +198,10 @@ jobs: chmod 440 etc/grid-security/hostcert.pem chmod 400 etc/grid-security/hostkey.pem - name: tests + env: + X509_CERT_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/lhcb.cern.ch/etc/grid-security/vomses run: | cd Pilot export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot @@ -205,4 +214,4 @@ jobs: sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json - python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb_d.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb-d.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug diff --git a/Pilot/pilotCommands.py b/Pilot/pilotCommands.py index 998f82dc..49fca074 100644 --- a/Pilot/pilotCommands.py +++ b/Pilot/pilotCommands.py @@ -196,9 +196,7 @@ def execute(self): self.exitWithError(1) class InstallDIRAC(CommandBase): - """Basically, this is used to call dirac-install with the passed parameters. - - It requires dirac-install script to be sitting in the same directory. + """ Source from CVMFS, or install locally """ def __init__(self, pilotParams): @@ -206,8 +204,6 @@ def __init__(self, pilotParams): super(InstallDIRAC, self).__init__(pilotParams) self.installOpts = [] self.pp.rootPath = self.pp.pilotRootPath - self.installScriptName = "dirac-install.py" - self.installScript = "" def _sourceEnvironmentFile(self): """source the $DIRAC_RC_FILE and save the created environment in self.pp.installEnv @@ -325,6 +321,9 @@ def _getPreinstalledEnvScript(self): def _installDIRACpy2(self): """ Install python2 DIRAC or its extension, then parse the environment file created, and use it for subsequent calls + + Basically, this is used to call dirac-install with the passed parameters. + It requires dirac-install script to be sitting in the same directory. """ # Installing @@ -458,6 +457,8 @@ def _installDIRACpy3(self): def _localInstallDIRAC(self): self.log.info("Installing DIRAC locally") if self.pp.pythonVersion == "27": + self.installScriptName = "dirac-install.py" + self.installScript = "" self._setInstallOptions() self._locateInstallationScript() self._installDIRACpy2() @@ -525,21 +526,6 @@ class ConfigureBasics(CommandBase): .. note:: Further commands should always call dirac-configure using the options -FDMH .. note:: If custom cfg file is created further commands should call dirac-configure with "-O %s %s" % ( self.pp.localConfigFile, self.pp.localConfigFile ) - - From here on, we have to pay attention to the paths. Specifically, we need to know where to look for - - * executables (scripts) - * DIRAC python code - - If the pilot has installed DIRAC (and extensions) in the traditional way, so using the dirac-install.py script, - simply the current directory is used, and: - - * scripts will be in $CWD/scripts. - * DIRAC python code will be all sitting in $CWD - * the local dirac.cfg file will be found in $CWD/etc - - For a more general case of non-traditional installations, we should use the PATH and PYTHONPATH as set by the - installation phase. Executables and code will be searched there. """ def __init__(self, pilotParams): @@ -609,14 +595,108 @@ def _getBasicsCFG(self): self.cfg.append('-o "/Resources/Computing/CEDefaults/VirtualOrganization=%s"' % self.pp.wnVO) def _getSecurityCFG(self): - """Nothing specific by default, but need to know host cert and key location in case they are needed""" + """ Sets security-related env variables, if needed + """ + # Need to know host cert and key location in case they are needed if self.pp.useServerCertificate: self.cfg.append("--UseServerCertificate") self.cfg.append("-o /DIRAC/Security/CertFile=%s/hostcert.pem" % self.pp.certsLocation) self.cfg.append("-o /DIRAC/Security/KeyFile=%s/hostkey.pem" % self.pp.certsLocation) + + # If DIRAC (or its extension) is installed in CVMFS: if self.pp.preinstalledEnv: - # Skip CAs download for preinstalled DIRAC - self.cfg.append("-D") + + if "X509_CERT_DIR" in os.environ: + self.log.debug( + "X509_CERT_DIR is set in the host environment as %s, aligning installEnv to it" + % os.environ["X509_CERT_DIR"] + ) + self.pp.installEnv["X509_CERT_DIR"] = os.environ["X509_CERT_DIR"] + else: + self.log.debug("X509_CERT_DIR is not set in the host environment") + # try and find it + for candidate in self.pp.CVMFS_locations: + self.log.debug( + "Candidate directory for X509_CERT_DIR is %s/etc/grid-security/certificates" + % candidate + ) + if os.path.isdir( + os.path.join( + os.path.expandvars(candidate), + '/etc/grid-security/certificates/' + ) + ): + self.log.debug("Setting X509_CERT_DIR=%s" % candidate) + self.pp.installEnv["X509_CERT_DIR"] = candidate + os.environ["X509_CERT_DIR"] = candidate + break + self.log.debug("%s not found or not a directory" % candidate) + + if "X509_CERT_DIR" not in self.pp.installEnv: + self.log.error("Could not find/set X509_CERT_DIR") + sys.exit(1) + + if "X509_VOMS_DIR" in os.environ: + self.log.debug( + "X509_VOMS_DIR is set in the host environment as %s, aligning installEnv to it" + % os.environ["X509_VOMS_DIR"] + ) + self.pp.installEnv["X509_VOMS_DIR"] = os.environ["X509_VOMS_DIR"] + else: + self.log.debug("X509_VOMS_DIR is not set in the host environment") + # try and find it + for candidate in self.pp.CVMFS_locations: + self.log.debug( + "Candidate directory for X509_VOMS_DIR is %s/etc/grid-security/vomsdir/" + % candidate + ) + if os.path.isdir( + os.path.join( + os.path.expandvars(candidate), + '/etc/grid-security/vomsdir/' + ) + ): + self.log.debug("Setting X509_VOMS_DIR=%s" % candidate) + self.pp.installEnv["X509_VOMS_DIR"] = candidate + os.environ["X509_VOMS_DIR"] = candidate + break + self.log.debug("%s not found" % candidate) + + if "X509_VOMS_DIR" not in self.pp.installEnv: + self.log.error("Could not find/set X509_VOMS_DIR") + sys.exit(1) + + if "DIRAC_VOMSES" in os.environ: + self.log.debug( + "DIRAC_VOMSES is set in the host environment as %s, aligning installEnv to it" + % os.environ["DIRAC_VOMSES"] + ) + self.pp.installEnv["DIRAC_VOMSES"] = os.environ["DIRAC_VOMSES"] + else: + self.log.debug("DIRAC_VOMSES is not set in the host environment") + # try and find it + for candidate in self.pp.CVMFS_locations: + self.log.debug( + "Candidate directory for DIRAC_VOMSES is %s/etc/grid-security/vomses/" + % candidate) + if os.path.isdir( + os.path.join( + os.path.expandvars(candidate), + '/etc/grid-security/vomses/' + ) + ): + self.log.debug("Setting DIRAC_VOMSES=%s" % candidate) + self.pp.installEnv["DIRAC_VOMSES"] = candidate + os.environ["DIRAC_VOMSES"] = candidate + break + self.log.debug("%s not found" % candidate) + + if "DIRAC_VOMSES" not in self.pp.installEnv: + self.log.error("Could not find/set DIRAC_VOMSES") + sys.exit(1) + + # In any case do not download VOMS and CAs + self.cfg.append("-DMH") class RegisterPilot(CommandBase): diff --git a/Pilot/pilotTools.py b/Pilot/pilotTools.py index fa349f25..fa215334 100644 --- a/Pilot/pilotTools.py +++ b/Pilot/pilotTools.py @@ -808,6 +808,10 @@ def __init__(self): self.modules = "" # see dirac-install "-m" option documentation self.userEnvVariables = "" # see dirac-install "--userEnvVariables" option documentation self.pipInstallOptions = "" + self.CVMFS_locations = [ + "/cvmfs/grid.cern.ch", + "/cvmfs/dirac.egi.eu", + ] # Parameters that can be determined at runtime only self.queueParameters = {} # from CE description From da0168809e64d4dd5368b83b32f44dd7b1b1e602 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 6 Oct 2023 15:57:46 +0200 Subject: [PATCH 8/8] test: added integration-cvmfs_matching test --- .github/workflows/integration.yml | 41 +++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a1cc31b4..f698c2ec 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -43,7 +43,6 @@ jobs: sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_version }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug - cat pilot.cfg integration-cvmfs: @@ -83,7 +82,6 @@ jobs: sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug - cat pilot.cfg matching: @@ -125,6 +123,45 @@ jobs: python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + integration-cvmfs_matching: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: cvmfs-contrib/github-action-cvmfs@v3 + + - name: Test CernVM-FS + run: ls /cvmfs/dirac.egi.eu + + - name: Retrieve the secret and decode it to a file + env: + HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }} + HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }} + run: | + cd Pilot + mkdir -p etc/grid-security + echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem + echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem + chmod 440 etc/grid-security/hostcert.pem + chmod 400 etc/grid-security/hostkey.pem + - name: tests + env: + X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates + X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir + DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses + run: | + cd Pilot + cp ../tests/CI/pilot_oldSchema.json pilot.json + sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json + sed -i "s/VAR_JENKINS_CE/jenkins-full.cern.ch/g" pilot.json + sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json + sed -i "s/VAR_DIRAC_VERSION/v8.1.0a16/g" pilot.json + sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json + sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json + sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json + python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug + + ################################## ### # extensions tests ##################################