Merge pull request #216 from fstagni/atsareg_envs

Fix for CVMFS locations lookup
DIRACGrid · Oct 31, 2023 · 4eb18f0 · 4eb18f0
2 parents ecaeec6 + e2d1498
commit 4eb18f0
Show file tree

Hide file tree

Showing 6 changed files with 291 additions and 146 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -56,6 +56,9 @@ jobs:
         pilot_schema:
           - pilot_oldSchema.json
           - pilot_newSchema.json
+        dirac_version:
+          - v8.0.27
+          - v8.1.0a17
 
     steps:
       - uses: actions/checkout@v4
@@ -86,11 +89,47 @@ jobs:
           sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
           sed -i "s/VAR_JENKINS_CE/jenkins.cern.ch/g" pilot.json
           sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
-          sed -i "s/VAR_DIRAC_VERSION/v8.1.0a16/g" pilot.json
+          sed -i "s/VAR_DIRAC_VERSION/${{ matrix.dirac_version }}/g" pilot.json
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam --debug
+
+
+  integration-cvmfs_no_env:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: cvmfs-contrib/github-action-cvmfs@v3
+
+      - name: Test CernVM-FS
+        run: ls /cvmfs/dirac.egi.eu
+
+      - name: Retrieve the secret and decode it to a file
+        env:
+          HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
+          HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
+        run: |
+          cd Pilot
+          mkdir -p etc/grid-security
+          echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem
+          echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem
+          chmod 440 etc/grid-security/hostcert.pem
+          chmod 400 etc/grid-security/hostkey.pem
+      - name: tests
+        run: |
+          cd Pilot
+          cp ../tests/CI/pilot_newSchema.json pilot.json
+          sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
+          sed -i "s/VAR_JENKINS_CE/jenkins.cern.ch/g" pilot.json
+          sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
+          sed -i "s/VAR_DIRAC_VERSION/v8.1.0a17/g" pilot.json
+          sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
+          sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
+          sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=dteam -o cvmfsOnly --debug
+
 
 
   matching:
@@ -138,6 +177,15 @@ jobs:
   integration-cvmfs_matching:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        VO: 
+          - dteam
+          - gridpp
+        dirac_version:
+          - v8.0.27
+          - v8.1.0a17
+
     steps:
       - uses: actions/checkout@v4
       - uses: cvmfs-contrib/github-action-cvmfs@v3
@@ -157,21 +205,17 @@ jobs:
           chmod 440 etc/grid-security/hostcert.pem
           chmod 400 etc/grid-security/hostkey.pem
       - name: tests
-        env:
-          X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates
-          X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir
-          DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses
         run: |
           cd Pilot
           cp ../tests/CI/pilot_newSchema.json pilot.json
           sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
           sed -i "s/VAR_JENKINS_CE/jenkins-full.cern.ch/g" pilot.json
           sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
-          sed -i "s/VAR_DIRAC_VERSION/v8.1.0a16/g" pilot.json
+          sed -i "s/VAR_DIRAC_VERSION/${{ matrix.dirac_version }}/g" pilot.json
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=${{ matrix.VO }} -o cvmfsOnly --debug
 
 
   ##################################
@@ -189,6 +233,9 @@ jobs:
         pilot_schema:
           - pilot_oldSchema.json
           - pilot_newSchema.json
+        pilot_version:
+          - master
+          - devel
 
     steps:
       - uses: actions/checkout@v4
@@ -203,9 +250,7 @@ jobs:
           HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
         run: |
           cd Pilot
-          mkdir -p etc/grid-security/vomses
-          mkdir -p etc/grid-security/vomsdir
-          mkdir -p etc/grid-security/certificates
+          mkdir -p etc/grid-security/
           echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem
           echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem
           chmod 440 etc/grid-security/hostcert.pem
@@ -218,7 +263,7 @@ jobs:
         run: |
           cd Pilot
           export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot
-          curl https://lhcbdirac.s3.cern.ch/Pilot3/LHCbPilotCommands.py -o LHCbPilotCommands.py
+          curl https://gitlab.cern.ch/lhcb-dirac/LHCbPilot/-/raw/${{ matrix.pilot_version }}/LHCbPilot/LHCbPilotCommands.py -o LHCbPilotCommands.py
           cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
           sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
           sed -i "s/VAR_JENKINS_CE/jenkins-lhcb.cern.ch/g" pilot.json
@@ -227,11 +272,20 @@ jobs:
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb  -o lbRunOnly --debug
 
-  ext-lhcb_integration_dirac_installer:
+  ext-lhcb_integration_dirac_installer_no_env:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        pilot_schema:
+          - pilot_oldSchema.json
+          - pilot_newSchema.json
+        ce:
+          - jenkins-lhcb-d.cern.ch
+          - jenkins-lhcb-dd.cern.ch
+
     steps:
       - uses: actions/checkout@v4
       - uses: cvmfs-contrib/github-action-cvmfs@v3
@@ -245,28 +299,65 @@ jobs:
           HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
         run: |
           cd Pilot
-          mkdir -p etc/grid-security/vomses
-          mkdir -p etc/grid-security/vomsdir
-          mkdir -p etc/grid-security/certificates
+          mkdir -p etc/grid-security/
           echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem
           echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem
           chmod 440 etc/grid-security/hostcert.pem
           chmod 400 etc/grid-security/hostkey.pem
       - name: tests
-        env:
-          X509_CERT_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/certificates
-          X509_VOMS_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir
-          DIRAC_VOMSES: /cvmfs/lhcb.cern.ch/etc/grid-security/vomses
         run: |
           cd Pilot
-          export VO_LHCB_SW_DIR=${GITHUB_WORKSPACE}/Pilot
           curl https://lhcbdirac.s3.cern.ch/Pilot3/LHCbPilotCommands.py -o LHCbPilotCommands.py
-          cp ../tests/CI/pilot_oldSchema.json pilot.json
+          cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
           sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
-          sed -i "s/VAR_JENKINS_CE/jenkins-lhcb.cern.ch/g" pilot.json
+          sed -i "s/VAR_JENKINS_CE/${{ matrix.ce }}/g" pilot.json
+          sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
+          sed -i "s/VAR_DIRAC_VERSION/v11.0.20/g" pilot.json
+          sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
+          sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
+          sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --debug
+
+  ext-lhcb_integration_no_env:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        pilot_schema:
+          - pilot_oldSchema.json
+          - pilot_newSchema.json
+        pilot_version:
+          - master
+          - devel
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: cvmfs-contrib/github-action-cvmfs@v3
+
+      - name: Test CernVM-FS
+        run: ls /cvmfs/lhcb.cern.ch
+
+      - name: Retrieve the secret and decode it to a file
+        env:
+          HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
+          HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
+        run: |
+          cd Pilot
+          mkdir -p etc/grid-security/
+          echo "$HOSTCERT_BASE64" | base64 --decode > etc/grid-security/hostcert.pem
+          echo "$HOSTKEY_BASE64" | base64 --decode > etc/grid-security/hostkey.pem
+          chmod 440 etc/grid-security/hostcert.pem
+          chmod 400 etc/grid-security/hostkey.pem
+      - name: tests
+        run: |
+          cd Pilot
+          curl https://gitlab.cern.ch/lhcb-dirac/LHCbPilot/-/raw/${{ matrix.pilot_version }}/LHCbPilot/LHCbPilotCommands.py -o LHCbPilotCommands.py
+          cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
+          sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
+          sed -i "s/VAR_JENKINS_CE/jenkins-lhcb-d.cern.ch/g" pilot.json
           sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
           sed -i "s/VAR_DIRAC_VERSION/v11.0.20/g" pilot.json
           sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
           sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
           sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
-          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb-d.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --debug
+          python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --cert --certLocation=${GITHUB_WORKSPACE}/Pilot/etc/grid-security --wnVO=LHCb -o cvmfsOnly --debug