Skip to content

Commit b302f5e

Browse files
fstagnifstagni
authored
Merge pull request #271 from fstagni/test_apptainer
test: integration test fixes
2 parents 1854581 + 56748e9 commit b302f5e

File tree

1 file changed

+31
-120
lines changed

1 file changed

+31
-120
lines changed

‎.github/workflows/integration.yml‎

Lines changed: 31 additions & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
pilot_schema:
1515
- pilot_oldSchema.json
1616
- pilot_newSchema.json
17-
VO:
17+
VO:
1818
- dteam
1919
- gridpp
2020

@@ -58,7 +58,7 @@ jobs:
5858
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
5959
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::${{ matrix.dirac_branch }} -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=${{ matrix.VO }} --pilotUUID="${pilotUUID}" --debug
6060
61-
release_prod_pre-cvmfs:
61+
release_prod_pre-cvmfs_CEs:
6262
runs-on: ubuntu-latest
6363

6464
strategy:
@@ -68,7 +68,13 @@ jobs:
6868
- pilot_newSchema.json
6969
dirac_version:
7070
- prod
71-
- pre
71+
- old
72+
ce:
73+
- jenkins.cern.ch
74+
- jenkins-singularity.cern.ch
75+
- jenkins-mp.cern.ch
76+
- jenkins-mp-pool.cern.ch
77+
- jenkins-mp-pool-singularity.cern.ch
7278

7379
steps:
7480
- uses: actions/checkout@v4
@@ -92,9 +98,9 @@ jobs:
9298
chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
9399
chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
94100
if [ "${{ matrix.dirac_version }}" == "prod" ]; then
95-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1)
101+
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1)
96102
else
97-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1)
103+
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v8' | tail -n 1)
98104
fi
99105
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
100106
pilot_proxy_file=$(mktemp)
@@ -112,59 +118,7 @@ jobs:
112118
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
113119
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
114120
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
115-
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --architectureScript=dirac-platform --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam --pilotUUID="${pilotUUID}" --debug
116-
117-
118-
release_prod-cvmfs_no_env_CEs:
119-
runs-on: ubuntu-latest
120-
121-
strategy:
122-
matrix:
123-
ce:
124-
- jenkins.cern.ch
125-
- jenkins-singularity.cern.ch
126-
- jenkins-mp.cern.ch
127-
- jenkins-mp-pool.cern.ch
128-
- jenkins-mp-pool-singularity.cern.ch
129-
- jenkins-revised.cern.ch
130-
131-
steps:
132-
- uses: actions/checkout@v4
133-
- uses: cvmfs-contrib/github-action-cvmfs@v3
134-
135-
- name: Test CernVM-FS
136-
run: ls /cvmfs/dirac.egi.eu
137-
138-
- name: Tests
139-
env:
140-
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
141-
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
142-
run: |
143-
mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security
144-
cd ${GITHUB_WORKSPACE}/Client
145-
echo "$HOSTCERT_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
146-
echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
147-
chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
148-
chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
149-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1)
150-
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
151-
pilot_proxy_file=$(mktemp)
152-
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
153-
dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
154-
cd ${GITHUB_WORKSPACE}/Pilot
155-
cp ../tests/CI/pilot_newSchema.json pilot.json
156-
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
157-
sed -i "s/VAR_JENKINS_CE/${{ matrix.ce }}/g" pilot.json
158-
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
159-
sed -i "s/VAR_DIRAC_VERSION/${version}/g" pilot.json
160-
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
161-
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
162-
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
163-
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
164-
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
165-
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
166-
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
167-
121+
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --architectureScript=dirac-platform --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=dteam --pilotUUID="${pilotUUID}" --debug
168122
169123
matching:
170124
runs-on: ubuntu-latest
@@ -186,14 +140,17 @@ jobs:
186140
env:
187141
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
188142
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
143+
X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates
144+
X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir
145+
DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses
189146
run: |
190147
mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security
191148
cd ${GITHUB_WORKSPACE}/Client
192149
echo "$HOSTCERT_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
193150
echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
194151
chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
195152
chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
196-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1)
153+
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1)
197154
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
198155
pilot_proxy_file=$(mktemp)
199156
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
@@ -212,18 +169,17 @@ jobs:
212169
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
213170
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py --modules https://github.com/DIRACGrid/DIRAC.git:::DIRAC:::integration -M 1 -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --wnVO=dteam --pilotUUID="${pilotUUID}" --debug
214171
215-
216172
release_prod_pre-cvmfs_matching:
217173
runs-on: ubuntu-latest
218174

219175
strategy:
220176
matrix:
221-
VO:
177+
VO:
222178
- dteam
223179
- gridpp
224180
dirac_version:
225181
- prod
226-
- pre
182+
- old
227183

228184
steps:
229185
- uses: actions/checkout@v4
@@ -236,6 +192,9 @@ jobs:
236192
env:
237193
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
238194
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
195+
X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates
196+
X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir
197+
DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses
239198
run: |
240199
mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security
241200
cd ${GITHUB_WORKSPACE}/Client
@@ -245,9 +204,9 @@ jobs:
245204
chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
246205
cd ${GITHUB_WORKSPACE}/Pilot
247206
if [ "${{ matrix.dirac_version }}" == "prod" ]; then
248-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1)
207+
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v9' | tail -n 1)
249208
else
250-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1)
209+
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'v8' | tail -n 1)
251210
fi
252211
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
253212
pilot_proxy_file=$(mktemp)
@@ -267,7 +226,6 @@ jobs:
267226
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
268227
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -S DIRAC-Certification -N jenkins-full.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --wnVO=${{ matrix.VO }} -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
269228
270-
271229
release_prod-cvmfs_matching_CEs:
272230
runs-on: ubuntu-latest
273231

@@ -279,7 +237,6 @@ jobs:
279237
- jenkins-mp-full.cern.ch
280238
- jenkins-mp-pool-full.cern.ch
281239
- jenkins-mp-pool-singularity-full.cern.ch
282-
- jenkins-revised.cern.ch
283240

284241
steps:
285242
- uses: actions/checkout@v4
@@ -292,14 +249,17 @@ jobs:
292249
env:
293250
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
294251
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
252+
X509_CERT_DIR: /cvmfs/grid.cern.ch/etc/grid-security/certificates
253+
X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir
254+
DIRAC_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses
295255
run: |
296256
mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security
297257
cd ${GITHUB_WORKSPACE}/Client
298258
echo "$HOSTCERT_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
299259
echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
300260
chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
301261
chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
302-
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep 'a' | tail -n 1)
262+
version=$(curl -s "https://api.github.com/repos/DIRACGrid/DIRAC/releases" | jq -r '.[].tag_name' | sort -V | grep -v 'a' | tail -n 1)
303263
source /cvmfs/dirac.egi.eu/dirac/${version}/Linux-x86_64/diracosrc
304264
pilot_proxy_file=$(mktemp)
305265
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
@@ -318,7 +278,6 @@ jobs:
318278
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
319279
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch --preinstalledEnvPrefix=/cvmfs/dirac.egi.eu/dirac --StopAfterFailedMatches=1 --wnVO=dteam -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
320280
321-
322281
##################################
323282
### # extensions tests
324283
##################################
@@ -380,58 +339,7 @@ jobs:
380339
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
381340
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -N jenkins-lhcb.cern.ch -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --architectureScript=dirac-architecture --wnVO=LHCb -o lbRunOnly --pilotUUID="${pilotUUID}" --debug
382341
383-
ext-lhcb_integration_dirac_installer_no_env:
384-
runs-on: ubuntu-latest
385-
386-
strategy:
387-
matrix:
388-
pilot_schema:
389-
- pilot_oldSchema.json
390-
- pilot_newSchema.json
391-
ce:
392-
- jenkins-lhcb-d.cern.ch
393-
- jenkins-lhcb-dd.cern.ch
394-
395-
steps:
396-
- uses: actions/checkout@v4
397-
- uses: cvmfs-contrib/github-action-cvmfs@v3
398-
399-
- name: Test CernVM-FS
400-
run: ls /cvmfs/lhcb.cern.ch
401-
402-
- name: Tests
403-
env:
404-
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
405-
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
406-
run: |
407-
mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security
408-
cd ${GITHUB_WORKSPACE}/Client
409-
echo "$HOSTCERT_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
410-
echo "$HOSTKEY_BASE64" | base64 --decode > ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
411-
chmod 440 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem
412-
chmod 400 ${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem
413-
version=$(echo $(curl -s "https://gitlab.cern.ch/api/v4/projects/3588/releases") | jq -r '.[].name' | sort -V | grep -v 'a' | tail -n 1)
414-
source /cvmfs/lhcb.cern.ch/lhcbdirac/${version}/Linux-x86_64/diracosrc
415-
pilot_proxy_file=$(mktemp)
416-
dirac-configure --ConfigurationServer=https://lbcertifdirac70.cern.ch:9135/Configuration/Server --UseServerCertificate -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --SkipCADownload -O config.cfg
417-
dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
418-
X509_CERT_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/certificates" X509_VOMS_DIR="/cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir" DIRAC_VOMSES="/cvmfs/lhcb.cern.ch/etc/grid-security/vomses" dirac-admin-get-proxy atsareg dteam_pilot -o /DIRAC/Security/UseServerCertificate=yes -o /DIRAC/Security/CertFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostcert.pem -o /DIRAC/Security/KeyFile=${GITHUB_WORKSPACE}/Client/etc/grid-security/hostkey.pem --out $pilot_proxy_file --cfg config.cfg
419-
cd ${GITHUB_WORKSPACE}/Pilot
420-
curl https://lhcbdirac.s3.cern.ch/Pilot3/LHCbPilotCommands.py -o LHCbPilotCommands.py
421-
cp ../tests/CI/${{ matrix.pilot_schema }} pilot.json
422-
sed -i "s/VAR_JENKINS_SITE/DIRAC.Jenkins.ch/g" pilot.json
423-
sed -i "s/VAR_JENKINS_CE/${{ matrix.ce }}/g" pilot.json
424-
sed -i "s/VAR_JENKINS_QUEUE/jenkins-queue_not_important/g" pilot.json
425-
sed -i "s/VAR_DIRAC_VERSION/${version}/g" pilot.json
426-
sed -i "s#VAR_CS#https://lbcertifdirac70.cern.ch:9135/Configuration/Server#g" pilot.json
427-
sed -i "s#VAR_USERDN#/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=alboyer/CN=819281/CN=Alexandre Franck Boyer#g" pilot.json
428-
sed -i "s#VAR_USERDN_GRIDPP#${DIRACUSERDN_GRIDPP}#g" pilot.json
429-
g_job="testintegrationworkflow${GITHUB_JOB//-/}"
430-
pilotUUID="${g_job//_/}""$(shuf -i 2000-65000 -n 1)"
431-
pilotUUID=$(echo $pilotUUID | rev | cut -c 1-32 | rev)
432-
X509_USER_PROXY=$pilot_proxy_file python dirac-pilot.py -M 1 -N ${{ matrix.ce }} -Q jenkins-queue_not_important -n DIRAC.Jenkins.ch -e LHCb -l LHCb -E LHCbPilot --preinstalledEnvPrefix=/cvmfs/lhcb.cern.ch/lhcbdirac/ --wnVO=LHCb -o cvmfsOnly --pilotUUID="${pilotUUID}" --debug
433-
434-
ext-lhcb_integration_no_env:
342+
ext-lhcb_integration_cvmfs:
435343
runs-on: ubuntu-latest
436344

437345
strategy:
@@ -454,6 +362,9 @@ jobs:
454362
env:
455363
HOSTCERT_BASE64: ${{ secrets.HOSTCERT_BASE64 }}
456364
HOSTKEY_BASE64: ${{ secrets.HOSTKEY_BASE64 }}
365+
X509_CERT_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/certificates
366+
X509_VOMS_DIR: /cvmfs/lhcb.cern.ch/etc/grid-security/vomsdir
367+
DIRAC_VOMSES: /cvmfs/lhcb.cern.ch/etc/grid-security/vomses
457368
run: |
458369
mkdir -p ${GITHUB_WORKSPACE}/Client/etc/grid-security
459370
cd ${GITHUB_WORKSPACE}/Client

0 commit comments

Comments
 (0)