From b6aef05ea03d687281a8cf940584490ac48cadab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Thu, 22 Jan 2026 12:32:34 +0100 Subject: [PATCH 01/10] Create role to set up CVMFS alien cache on NFS Configure the CVMFS client to use a tiered two-level cache, with the regular CVMFS disk cache as upper level and an alien cache on NFS as the lower level. To retrieve a chunk, the CVMFS client will first attempt to find it in the disk cache; if there's a cache miss, it will look for it in the NFS cache. The CVMFS clients are responsible for filling the NFS cache. > The alien cache directory [...] can be located anywhere including cluster and network file systems. If configured, all data chunks are stored there. CernVM-FS ensures atomic access to the cache directory. It is safe to have the alien directory shared by multiple CernVM-FS processes, and it is safe to unlink files from the alien cache directory anytime. The contents of files, however, must not be touched by third-party programs. --- .../defaults/main.yml | 4 +++ roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml | 26 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 roles/usegalaxy_eu.cvmfs_cache/defaults/main.yml create mode 100644 roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml diff --git a/roles/usegalaxy_eu.cvmfs_cache/defaults/main.yml b/roles/usegalaxy_eu.cvmfs_cache/defaults/main.yml new file mode 100644 index 000000000..dc034997e --- /dev/null +++ b/roles/usegalaxy_eu.cvmfs_cache/defaults/main.yml @@ -0,0 +1,4 @@ +--- +cvmfs_cache_cvmfs_config_file: /etc/cvmfs/default.local +cvmfs_cache_disk_quota_limit: 4000 +cmvfs_cache_nfs_cache_dir: /data/db/cvmfs_cache/ diff --git a/roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml b/roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml new file mode 100644 index 000000000..b44cca87f --- /dev/null +++ b/roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml @@ -0,0 +1,26 @@ +- name: Configure CVMFS cache + ansible.builtin.blockinfile: + path: "{{ cvmfs_cache_cvmfs_config_file }}" + block: | + CVMFS_CACHE_PRIMARY=cvmfs + + CVMFS_CACHE_cvmfs_TYPE=tiered + CVMFS_CACHE_cvmfs_UPPER=disk + CVMFS_CACHE_cvmfs_UPPER_READONLY=no + CVMFS_CACHE_cvmfs_LOWER=nfs + CVMFS_CACHE_cvmfs_LOWER_READONLY=no + + CVMFS_CACHE_disk_TYPE=posix + CVMFS_CACHE_disk_QUOTA_LIMIT={{ cvmfs_cache_disk_quota_limit }} + + CVMFS_CACHE_nfs_TYPE=posix + CVMFS_CACHE_nfs_ALIEN={{ cmvfs_cache_nfs_cache_dir }} + CVMFS_CACHE_nfs_SHARED=no + CVMFS_CACHE_nfs_QUOTA_LIMIT=-1 + marker: "# {mark} ANSIBLE MANAGED BLOCK (`usegalaxy_eu.cvmfs_cache` CVMFS cache)" + prepend_newline: true + append_newline: true + state: present + create: false + +# TODO: restart/reload CVMFS From c22798f235bd3bc504098cc2b035bec27e3b8761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Thu, 22 Jan 2026 12:33:28 +0100 Subject: [PATCH 02/10] Use `usegalaxy_eu.cvmfs_cache` to set up CVMFS alien cache on sn07 and sn09 --- group_vars/sn07.yml | 2 ++ group_vars/sn09/sn09.yml | 2 ++ sn07.yml | 1 + sn09.yml | 1 + 4 files changed, 6 insertions(+) diff --git a/group_vars/sn07.yml b/group_vars/sn07.yml index 74977aa02..8e3a03add 100644 --- a/group_vars/sn07.yml +++ b/group_vars/sn07.yml @@ -243,6 +243,8 @@ galaxy_cvmfs_server_urls: urls: - "http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@" +cvmfs_cache_disk_quota_limit: 50000 + # SystemD galaxy_systemd_mode: "gunicorn" galaxy_systemd_gunicorns: 2 diff --git a/group_vars/sn09/sn09.yml b/group_vars/sn09/sn09.yml index 8f1329738..5359d090c 100644 --- a/group_vars/sn09/sn09.yml +++ b/group_vars/sn09/sn09.yml @@ -231,6 +231,8 @@ galaxy_cvmfs_server_urls: urls: - 'http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@' +cvmfs_cache_disk_quota_limit: 50000 + # SystemD galaxy_systemd_mode: 'gunicorn' galaxy_systemd_gunicorns: 7 diff --git a/sn07.yml b/sn07.yml index fb4f921a8..43da5967b 100644 --- a/sn07.yml +++ b/sn07.yml @@ -166,6 +166,7 @@ ## Filesystems - usegalaxy-eu.autofs # Setup the mount points which will be needed later - galaxyproject.cvmfs # Galaxy datasets + - usegalaxy_eu.cvmfs_cache ## Monitoring # this should be unique, because it sends data to about the cluster to influxDB diff --git a/sn09.yml b/sn09.yml index 44757b1f7..d253fb4e9 100644 --- a/sn09.yml +++ b/sn09.yml @@ -264,6 +264,7 @@ ## Filesystems - usegalaxy-eu.autofs # Setup the mount points which will be needed later - galaxyproject.cvmfs # Galaxy datasets + - usegalaxy_eu.cvmfs_cache ## Monitoring - hxr.monitor-email From 83d52136042662f0eca57735ed34291f6ff3cff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Thu, 29 Jan 2026 15:39:57 +0100 Subject: [PATCH 03/10] Setup CVMFS alien cache using `galaxyproject.cvmfs` Switch from the `usegalaxy_eu.cvmfs_cache` role to a version of the `galaxyproject.cvmfs` role supporting the configuration of arbitrary CVMFS parameters. --- group_vars/cvmfs_clients.yml | 24 ++++++++++++++++++++++++ group_vars/sn07.yml | 11 ----------- group_vars/sn09/sn09.yml | 11 ----------- hosts | 4 ++++ sn07.yml | 1 - sn09.yml | 1 - 6 files changed, 28 insertions(+), 24 deletions(-) create mode 100644 group_vars/cvmfs_clients.yml diff --git a/group_vars/cvmfs_clients.yml b/group_vars/cvmfs_clients.yml new file mode 100644 index 000000000..fb20316ad --- /dev/null +++ b/group_vars/cvmfs_clients.yml @@ -0,0 +1,24 @@ +cvmfs_role: client + +cvmfs_params: + CVMFS_CACHE_PRIMARY: cvmfs + + CVMFS_CACHE_cvmfs_TYPE: tiered + CVMFS_CACHE_cvmfs_UPPER: disk + CVMFS_CACHE_cvmfs_UPPER_READONLY: no + CVMFS_CACHE_cvmfs_LOWER: nfs + CVMFS_CACHE_cvmfs_LOWER_READONLY: no + + CVMFS_CACHE_disk_TYPE: posix + CVMFS_CACHE_disk_QUOTA_LIMIT: 50000 + + CVMFS_CACHE_nfs_TYPE: posix + CVMFS_CACHE_nfs_ALIEN: /data/db/cvmfs_cache/ + CVMFS_CACHE_nfs_SHARED: no + CVMFS_CACHE_nfs_QUOTA_LIMIT: -1 + +galaxy_cvmfs_repos_enabled: config-repo +galaxy_cvmfs_server_urls: + - domain: galaxyproject.org + urls: + - 'http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@' diff --git a/group_vars/sn07.yml b/group_vars/sn07.yml index 8e3a03add..69f49f0a1 100644 --- a/group_vars/sn07.yml +++ b/group_vars/sn07.yml @@ -234,17 +234,6 @@ dynmotd_custom: # TPV Script tpv_config_dir_name: total_perspective_vortex -# CVMFS -cvmfs_role: client -galaxy_cvmfs_repos_enabled: config-repo - -galaxy_cvmfs_server_urls: - - domain: galaxyproject.org - urls: - - "http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@" - -cvmfs_cache_disk_quota_limit: 50000 - # SystemD galaxy_systemd_mode: "gunicorn" galaxy_systemd_gunicorns: 2 diff --git a/group_vars/sn09/sn09.yml b/group_vars/sn09/sn09.yml index 5359d090c..f3ffe1ee6 100644 --- a/group_vars/sn09/sn09.yml +++ b/group_vars/sn09/sn09.yml @@ -222,17 +222,6 @@ dynmotd_custom: # TPV Script tpv_config_dir_name: total_perspective_vortex -# CVMFS -cvmfs_role: client -galaxy_cvmfs_repos_enabled: config-repo - -galaxy_cvmfs_server_urls: - - domain: galaxyproject.org - urls: - - 'http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@' - -cvmfs_cache_disk_quota_limit: 50000 - # SystemD galaxy_systemd_mode: 'gunicorn' galaxy_systemd_gunicorns: 7 diff --git a/hosts b/hosts index c1502543a..bb49a83a1 100644 --- a/hosts +++ b/hosts @@ -42,6 +42,10 @@ cvmfs1-ufr0.internal.galaxyproject.eu [cvmfs-stratum0] cvmfs-stratum0.galaxyproject.eu +[cvmfs_clients:children] +sn07 +sn09 + [plausible] plausible.galaxyproject.eu diff --git a/sn07.yml b/sn07.yml index 43da5967b..fb4f921a8 100644 --- a/sn07.yml +++ b/sn07.yml @@ -166,7 +166,6 @@ ## Filesystems - usegalaxy-eu.autofs # Setup the mount points which will be needed later - galaxyproject.cvmfs # Galaxy datasets - - usegalaxy_eu.cvmfs_cache ## Monitoring # this should be unique, because it sends data to about the cluster to influxDB diff --git a/sn09.yml b/sn09.yml index d253fb4e9..44757b1f7 100644 --- a/sn09.yml +++ b/sn09.yml @@ -264,7 +264,6 @@ ## Filesystems - usegalaxy-eu.autofs # Setup the mount points which will be needed later - galaxyproject.cvmfs # Galaxy datasets - - usegalaxy_eu.cvmfs_cache ## Monitoring - hxr.monitor-email From ebb4f0c429a00e9204d95eeb5762e51694075d5e Mon Sep 17 00:00:00 2001 From: Gabriel Saudade <155427716+gsaudade99@users.noreply.github.com> Date: Thu, 29 Jan 2026 16:56:18 +0100 Subject: [PATCH 04/10] Update main.yml --- roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml b/roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml index b44cca87f..33983d935 100644 --- a/roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml +++ b/roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml @@ -14,7 +14,7 @@ CVMFS_CACHE_disk_QUOTA_LIMIT={{ cvmfs_cache_disk_quota_limit }} CVMFS_CACHE_nfs_TYPE=posix - CVMFS_CACHE_nfs_ALIEN={{ cmvfs_cache_nfs_cache_dir }} + CVMFS_CACHE_nfs_ALIEN={{ cvmfs_cache_nfs_cache_dir }} CVMFS_CACHE_nfs_SHARED=no CVMFS_CACHE_nfs_QUOTA_LIMIT=-1 marker: "# {mark} ANSIBLE MANAGED BLOCK (`usegalaxy_eu.cvmfs_cache` CVMFS cache)" From 673f620436288106e3a1f0b257f681f68f233008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Thu, 29 Jan 2026 16:58:38 +0100 Subject: [PATCH 05/10] Remove role `usegalaxy_eu.cvmfs_cache` --- roles/usegalaxy_eu.cvmfs_cache/defaults/main.yml | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 roles/usegalaxy_eu.cvmfs_cache/defaults/main.yml diff --git a/roles/usegalaxy_eu.cvmfs_cache/defaults/main.yml b/roles/usegalaxy_eu.cvmfs_cache/defaults/main.yml deleted file mode 100644 index dc034997e..000000000 --- a/roles/usegalaxy_eu.cvmfs_cache/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -cvmfs_cache_cvmfs_config_file: /etc/cvmfs/default.local -cvmfs_cache_disk_quota_limit: 4000 -cmvfs_cache_nfs_cache_dir: /data/db/cvmfs_cache/ From 2dbeb2bb790e3afca82f3e6bcda6185a83ca567a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Tue, 3 Feb 2026 11:11:19 +0100 Subject: [PATCH 06/10] Remove sn07 from cvmfs_clients group --- hosts | 1 - 1 file changed, 1 deletion(-) diff --git a/hosts b/hosts index bb49a83a1..c107e40e7 100644 --- a/hosts +++ b/hosts @@ -43,7 +43,6 @@ cvmfs1-ufr0.internal.galaxyproject.eu cvmfs-stratum0.galaxyproject.eu [cvmfs_clients:children] -sn07 sn09 [plausible] From 765b591b383a5ea1c50d6129dc2986ef4051bb76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Tue, 3 Feb 2026 11:14:49 +0100 Subject: [PATCH 07/10] Remove role `usegalaxy_eu.cvmfs_cache` The first attempt 673f620 was incomplete. --- roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml diff --git a/roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml b/roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml deleted file mode 100644 index 33983d935..000000000 --- a/roles/usegalaxy_eu.cvmfs_cache/tasks/main.yml +++ /dev/null @@ -1,26 +0,0 @@ -- name: Configure CVMFS cache - ansible.builtin.blockinfile: - path: "{{ cvmfs_cache_cvmfs_config_file }}" - block: | - CVMFS_CACHE_PRIMARY=cvmfs - - CVMFS_CACHE_cvmfs_TYPE=tiered - CVMFS_CACHE_cvmfs_UPPER=disk - CVMFS_CACHE_cvmfs_UPPER_READONLY=no - CVMFS_CACHE_cvmfs_LOWER=nfs - CVMFS_CACHE_cvmfs_LOWER_READONLY=no - - CVMFS_CACHE_disk_TYPE=posix - CVMFS_CACHE_disk_QUOTA_LIMIT={{ cvmfs_cache_disk_quota_limit }} - - CVMFS_CACHE_nfs_TYPE=posix - CVMFS_CACHE_nfs_ALIEN={{ cvmfs_cache_nfs_cache_dir }} - CVMFS_CACHE_nfs_SHARED=no - CVMFS_CACHE_nfs_QUOTA_LIMIT=-1 - marker: "# {mark} ANSIBLE MANAGED BLOCK (`usegalaxy_eu.cvmfs_cache` CVMFS cache)" - prepend_newline: true - append_newline: true - state: present - create: false - -# TODO: restart/reload CVMFS From de537c376e2ea668aa3cbc23a83d88163bdb1390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Tue, 3 Feb 2026 14:22:40 +0100 Subject: [PATCH 08/10] Set `CVMFS_MEMCACHE_SIZE` to `256` The setting may either slightly improve performance or have no noticeable effect. But it won't hurt. > The default settings in CernVM-FS are tailored to the normal, non-NFS use case. For decent performance in the NFS deployment, the amount of memory given to the metadata cache should be increased. By default, this is 16M. It can be increased, for instance, to 256M by setting `CVMFS_MEMCACHE_SIZE` to 256. Furthermore, the maximum number of download retries should be increased to at least 2. See https://cvmfs.readthedocs.io/en/2.13/cpt-configure.html#tuning. --- group_vars/cvmfs_clients.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/group_vars/cvmfs_clients.yml b/group_vars/cvmfs_clients.yml index fb20316ad..c324a284d 100644 --- a/group_vars/cvmfs_clients.yml +++ b/group_vars/cvmfs_clients.yml @@ -1,6 +1,8 @@ cvmfs_role: client cvmfs_params: + CVMFS_MEMCACHE_SIZE: 256 + CVMFS_CACHE_PRIMARY: cvmfs CVMFS_CACHE_cvmfs_TYPE: tiered From aeccbf516040dd5efde636824670a59bbf8af4b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Tue, 3 Feb 2026 14:26:09 +0100 Subject: [PATCH 09/10] Set `CVMFS_CACHE_disk_QUOTA_LIMIT` to `5000` It doesn't look like the headnode is reading whole files at all from CVMFS. Nevertheless, some cache space is needed for the FS metadata (which is relatively large) but a few GB should suffice. The default is 4GB, make it 5GB to give the system slightly more room, as at the moment the headnode is using ~3GiB, although it does not seem to be trashing. --- group_vars/cvmfs_clients.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/cvmfs_clients.yml b/group_vars/cvmfs_clients.yml index c324a284d..418e7dc06 100644 --- a/group_vars/cvmfs_clients.yml +++ b/group_vars/cvmfs_clients.yml @@ -12,7 +12,7 @@ cvmfs_params: CVMFS_CACHE_cvmfs_LOWER_READONLY: no CVMFS_CACHE_disk_TYPE: posix - CVMFS_CACHE_disk_QUOTA_LIMIT: 50000 + CVMFS_CACHE_disk_QUOTA_LIMIT: 5000 CVMFS_CACHE_nfs_TYPE: posix CVMFS_CACHE_nfs_ALIEN: /data/db/cvmfs_cache/ From 8954203766db53445c23b0325fdeb1c0000f86f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Manuel=20Dom=C3=ADnguez?= Date: Mon, 9 Feb 2026 15:46:40 +0100 Subject: [PATCH 10/10] Update path for CVMFS alien cache on NFS Use the new cvmfs08 share to store the CVMFS alien cache on NFS. --- group_vars/cvmfs_clients.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/cvmfs_clients.yml b/group_vars/cvmfs_clients.yml index 418e7dc06..aa5c936d2 100644 --- a/group_vars/cvmfs_clients.yml +++ b/group_vars/cvmfs_clients.yml @@ -15,7 +15,7 @@ cvmfs_params: CVMFS_CACHE_disk_QUOTA_LIMIT: 5000 CVMFS_CACHE_nfs_TYPE: posix - CVMFS_CACHE_nfs_ALIEN: /data/db/cvmfs_cache/ + CVMFS_CACHE_nfs_ALIEN: /data/cvmfs08/cache/ CVMFS_CACHE_nfs_SHARED: no CVMFS_CACHE_nfs_QUOTA_LIMIT: -1