From 846e5a77b1cb89dd92145f00af27243de7cdc1cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Oct 2023 16:20:00 +0200 Subject: [PATCH 01/13] add role for setting cvmfs server meta info --- roles/cvmfs_server_meta_info/defaults/main.yml | 7 +++++++ roles/cvmfs_server_meta_info/tasks/main.yml | 10 ++++++++++ 2 files changed, 17 insertions(+) create mode 100644 roles/cvmfs_server_meta_info/defaults/main.yml create mode 100644 roles/cvmfs_server_meta_info/tasks/main.yml diff --git a/roles/cvmfs_server_meta_info/defaults/main.yml b/roles/cvmfs_server_meta_info/defaults/main.yml new file mode 100644 index 00000000..8d10564c --- /dev/null +++ b/roles/cvmfs_server_meta_info/defaults/main.yml @@ -0,0 +1,7 @@ +--- +cvmfs_metadata: + administrator: "{{ cvmfs_metadata_administrator | default('Your Name') }}" + email: "{{ cvmfs_metadata_email | default('you@organisation.org') }}" + organisation: "{{ cvmfs_metadata_organisation | default('Your Organisation') }}" + custom: "{{ cvmfs_metadata_custom | default({'_comment': 'Put arbitrary structured data here'}) }}" +... diff --git a/roles/cvmfs_server_meta_info/tasks/main.yml b/roles/cvmfs_server_meta_info/tasks/main.yml new file mode 100644 index 00000000..f42dca9e --- /dev/null +++ b/roles/cvmfs_server_meta_info/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: Create a meta.json containing the CVMFS Server Meta Information + ansible.builtin.copy: + content: "{{ cvmfs_metadata | to_nice_json(indent=2, sort_keys=false) }}" + dest: "/srv/cvmfs/info/v1/meta.json" + owner: root + group: root + mode: 0644 + become: true +... From ebd9dd01e16243260124d6af4425e0f4103bbddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Oct 2023 16:21:17 +0200 Subject: [PATCH 02/13] invoke cvmfs_server_meta_info role on stratums --- stratum0.yml | 2 ++ stratum1.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/stratum0.yml b/stratum0.yml index 0e234861..92a49c3c 100644 --- a/stratum0.yml +++ b/stratum0.yml @@ -16,3 +16,5 @@ - role: geerlingguy.repo-epel when: ansible_facts['os_family'] == 'RedHat' - galaxyproject.cvmfs + - cvmfs_server_meta_info +... diff --git a/stratum1.yml b/stratum1.yml index 9edcde3c..fc605256 100644 --- a/stratum1.yml +++ b/stratum1.yml @@ -17,3 +17,5 @@ - role: geerlingguy.repo-epel when: ansible_facts['os_family'] == 'RedHat' - galaxyproject.cvmfs + - cvmfs_server_meta_info +... From dca8fd48160bf72f8fceb24ce7e8eaa1b4bd3439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Oct 2023 16:22:11 +0200 Subject: [PATCH 03/13] add meta info for our stratums --- inventory/group_vars/all.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 1ff0b8bf..28667ee9 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -107,6 +107,13 @@ eessi_cvmfs_repositories: client_options: [] use_for_ci: true +# Meta information for CVMFS Stratum servers +cvmfs_meta_administrator: "EESSI CVMFS Administrators" +cvmfs_meta_email: "{{ eessi_email }}" +cvmfs_meta_organisation: "EESSI" +cvmfs_meta_custom: + _comment: "See https://eessi.io/docs/ for more information about the EESSI repository." + # Override all the Galaxy defaults by our EESSI defaults. # This is required, beucase the galaxy_* variables are used inside the Ansible tasks. galaxy_cvmfs_config_repo: "{{ eessi_cvmfs_config_repo }}" From fe247cbf500f71abe9e779a832bde9083947b41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Oct 2023 16:23:16 +0200 Subject: [PATCH 04/13] rename variables --- roles/cvmfs_server_meta_info/defaults/main.yml | 10 +++++----- roles/cvmfs_server_meta_info/tasks/main.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/cvmfs_server_meta_info/defaults/main.yml b/roles/cvmfs_server_meta_info/defaults/main.yml index 8d10564c..8c0f61cd 100644 --- a/roles/cvmfs_server_meta_info/defaults/main.yml +++ b/roles/cvmfs_server_meta_info/defaults/main.yml @@ -1,7 +1,7 @@ --- -cvmfs_metadata: - administrator: "{{ cvmfs_metadata_administrator | default('Your Name') }}" - email: "{{ cvmfs_metadata_email | default('you@organisation.org') }}" - organisation: "{{ cvmfs_metadata_organisation | default('Your Organisation') }}" - custom: "{{ cvmfs_metadata_custom | default({'_comment': 'Put arbitrary structured data here'}) }}" +cvmfs_meta: + administrator: "{{ cvmfs_meta_administrator | default('Your Name') }}" + email: "{{ cvmfs_meta_email | default('you@organisation.org') }}" + organisation: "{{ cvmfs_meta_organisation | default('Your Organisation') }}" + custom: "{{ cvmfs_meta_custom | default({'_comment': 'Put arbitrary structured data here'}) }}" ... diff --git a/roles/cvmfs_server_meta_info/tasks/main.yml b/roles/cvmfs_server_meta_info/tasks/main.yml index f42dca9e..8b16b571 100644 --- a/roles/cvmfs_server_meta_info/tasks/main.yml +++ b/roles/cvmfs_server_meta_info/tasks/main.yml @@ -1,7 +1,7 @@ --- - name: Create a meta.json containing the CVMFS Server Meta Information ansible.builtin.copy: - content: "{{ cvmfs_metadata | to_nice_json(indent=2, sort_keys=false) }}" + content: "{{ cvmfs_meta | to_nice_json(indent=2, sort_keys=false) }}" dest: "/srv/cvmfs/info/v1/meta.json" owner: root group: root From 96257e0102276bbb6edaa7c5a16b60abd24647ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Oct 2023 16:24:00 +0200 Subject: [PATCH 05/13] update EESSI email --- inventory/group_vars/all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 28667ee9..df15b030 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -5,7 +5,7 @@ eessi_cvmfs_repos_enabled: true # Email address for the project. -eessi_email: eessi@list.rug.nl +eessi_email: support@eessi.io # # Defaults for eessi-hpc.org repos. From 89d774c9de5384b0fa4487023a5ae3487d41ef00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 6 Oct 2023 23:05:57 +0200 Subject: [PATCH 06/13] add descriptions to repos, rename some variables --- inventory/group_vars/all.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index df15b030..2fcae6ea 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -87,6 +87,7 @@ eessi_cvmfs_repositories: - CVMFS_GARBAGE_COLLECTION=true client_options: [] use_for_ci: false + description: Repository containing pilot versions of the EESSI software stack. - repository: data.eessi-hpc.org stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org owner: "{{ cvmfs_repo_owner | default('root') }}" @@ -97,6 +98,7 @@ eessi_cvmfs_repositories: - CVMFS_GARBAGE_COLLECTION=true client_options: [] use_for_ci: false + description: "Data files used for applications available in the EESSI stack." - repository: ci.eessi-hpc.org stratum0: rug-nl.stratum0.cvmfs.eessi-infra.org owner: "{{ cvmfs_repo_owner | default('root') }}" @@ -106,13 +108,15 @@ eessi_cvmfs_repositories: - CVMFS_GARBAGE_COLLECTION=true client_options: [] use_for_ci: true + description: "Small-sized epository used for CI purposes." # Meta information for CVMFS Stratum servers -cvmfs_meta_administrator: "EESSI CVMFS Administrators" -cvmfs_meta_email: "{{ eessi_email }}" -cvmfs_meta_organisation: "EESSI" -cvmfs_meta_custom: +cvmfs_server_meta_administrator: "EESSI CVMFS Administrators" +cvmfs_server_meta_email: "{{ eessi_email }}" +cvmfs_server_meta_organisation: "EESSI" +cvmfs_server_meta_custom: _comment: "See https://eessi.io/docs/ for more information about the EESSI repository." +cvmfs_server_meta_website: "https://eessi.io" # Override all the Galaxy defaults by our EESSI defaults. # This is required, beucase the galaxy_* variables are used inside the Ansible tasks. From caa0675aff4c3bc4d844ba791203360d5616d482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Oct 2023 14:04:53 +0200 Subject: [PATCH 07/13] add step for looping over repos --- roles/cvmfs_server_meta_info/tasks/main.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/roles/cvmfs_server_meta_info/tasks/main.yml b/roles/cvmfs_server_meta_info/tasks/main.yml index 8b16b571..ee5bed33 100644 --- a/roles/cvmfs_server_meta_info/tasks/main.yml +++ b/roles/cvmfs_server_meta_info/tasks/main.yml @@ -1,10 +1,22 @@ --- - name: Create a meta.json containing the CVMFS Server Meta Information ansible.builtin.copy: - content: "{{ cvmfs_meta | to_nice_json(indent=2, sort_keys=false) }}" - dest: "/srv/cvmfs/info/v1/meta.json" + content: "{{ cvmfs_server_meta | to_nice_json(indent=2, sort_keys=false) }}" + dest: "{{ cvmfs_srv_mount }}/cvmfs/info/v1/meta.json" owner: root group: root mode: 0644 become: true + +- name: Create meta information for each CVMFS repository + ansible.builtin.include_tasks: repo_meta_info.yml + vars: + this_cvmfs_repo: "{{ item }}" + this_cvmfs_repo_meta: + description: "{{ item.description | default(omit) }}" + recommended-stratum0: "http://{{ item.stratum0 }}/cvmfs/{{ item.repository }}" + recommended-stratum1s: "{{ eessi_cvmfs_server_urls | selectattr('domain', 'in', item.repository) | + map(attribute='urls') | map('regex_replace', '@fqrn@', item.repository)}}" + with_items: "{{ eessi_cvmfs_repositories }}" + when: "'cvmfsstratum0servers' in group_names or cvmfs_role == 'stratum0'" ... From 526bd39c1caae412841683aba4058b05b5727dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Oct 2023 14:05:25 +0200 Subject: [PATCH 08/13] update meta info of a given repo --- .../tasks/repo_meta_info.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml diff --git a/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml b/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml new file mode 100644 index 00000000..20afe2a8 --- /dev/null +++ b/roles/cvmfs_server_meta_info/tasks/repo_meta_info.yml @@ -0,0 +1,35 @@ +--- +- name: Create temporary file for storing json + ansible.builtin.tempfile: + state: file + suffix: .json + register: tmp_json_file + +- name: Add meta information for repository to temporary json file + ansible.builtin.copy: + content: "{{ cvmfs_repo_meta | combine(this_cvmfs_repo_meta) | to_nice_json(indent=2, sort_keys=false) }}" + dest: "{{ tmp_json_file.path }}" + mode: 0644 + +- name: Calculate the checksum of the json file + ansible.builtin.stat: + path: "{{ tmp_json_file.path }}" + register: json_file_stat + +- name: Get the current meta information of this repository from the server + ansible.builtin.command: + cmd: "cvmfs_swissknife info -M -r {{ this_cvmfs_repo_meta['recommended-stratum0'] }}" + changed_when: false + register: current_repo_meta + +- name: Update the repository's meta information + ansible.builtin.command: + cmd: "cvmfs_server update-repoinfo -f {{ tmp_json_file.path }} {{ this_cvmfs_repo.repository }}" + when: (current_repo_meta.stdout | checksum) != json_file_stat.stat.checksum + become_user: "{{ cvmfs_repo_owner }}" + +- name: Remove temporary json file + ansible.builtin.file: + path: "{{ tmp_json_file.path }}" + state: absent +... From 28f387283a97aa1a199474bfa46cccf6c75fa3c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Oct 2023 14:05:38 +0200 Subject: [PATCH 09/13] add defaults for repo meta info --- .../cvmfs_server_meta_info/defaults/main.yml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/roles/cvmfs_server_meta_info/defaults/main.yml b/roles/cvmfs_server_meta_info/defaults/main.yml index 8c0f61cd..679fc452 100644 --- a/roles/cvmfs_server_meta_info/defaults/main.yml +++ b/roles/cvmfs_server_meta_info/defaults/main.yml @@ -1,7 +1,18 @@ --- -cvmfs_meta: - administrator: "{{ cvmfs_meta_administrator | default('Your Name') }}" - email: "{{ cvmfs_meta_email | default('you@organisation.org') }}" - organisation: "{{ cvmfs_meta_organisation | default('Your Organisation') }}" - custom: "{{ cvmfs_meta_custom | default({'_comment': 'Put arbitrary structured data here'}) }}" +cvmfs_server_meta: + administrator: "{{ cvmfs_server_meta_administrator | default('Your Name') }}" + email: "{{ cvmfs_server_meta_email | default('you@organisation.org') }}" + organisation: "{{ cvmfs_server_meta_organisation | default('Your Organisation') }}" + custom: "{{ cvmfs_server_meta_custom | default({'_comment': 'Put arbitrary structured data here'}) }}" + +cvmfs_repo_meta: + administrator: "{{ cvmfs_server_meta_administrator | default('Your Name') }}" + email: "{{ cvmfs_server_meta_email | default('you@organisation.org') }}" + organisation: "{{ cvmfs_server_meta_organisation | default('Your Organisation') }}" + description: "Repository content" + url: "{{ cvmfs_server_meta_website | default('Project website') }}" + recommended-stratum0: "stratum 0 url" + recommended-stratum1s: [ "stratum1 url", "stratum1 url" ] + + custom: "{{ cvmfs_server_meta_custom | default({'_comment': 'Put arbitrary structured data here'}) }}" ... From e63cf08eaf308f05bf5d10da742d19702bb1bc23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Oct 2023 14:17:24 +0200 Subject: [PATCH 10/13] remove spaces --- roles/cvmfs_server_meta_info/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cvmfs_server_meta_info/defaults/main.yml b/roles/cvmfs_server_meta_info/defaults/main.yml index 679fc452..91b0a272 100644 --- a/roles/cvmfs_server_meta_info/defaults/main.yml +++ b/roles/cvmfs_server_meta_info/defaults/main.yml @@ -12,7 +12,7 @@ cvmfs_repo_meta: description: "Repository content" url: "{{ cvmfs_server_meta_website | default('Project website') }}" recommended-stratum0: "stratum 0 url" - recommended-stratum1s: [ "stratum1 url", "stratum1 url" ] + recommended-stratum1s: ["stratum1 url", "stratum1 url"] custom: "{{ cvmfs_server_meta_custom | default({'_comment': 'Put arbitrary structured data here'}) }}" ... From c3f965a6681ecabf5a68636e577d826c81099a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Oct 2023 14:17:44 +0200 Subject: [PATCH 11/13] remove another space --- roles/cvmfs_server_meta_info/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cvmfs_server_meta_info/defaults/main.yml b/roles/cvmfs_server_meta_info/defaults/main.yml index 91b0a272..6bf29641 100644 --- a/roles/cvmfs_server_meta_info/defaults/main.yml +++ b/roles/cvmfs_server_meta_info/defaults/main.yml @@ -11,7 +11,7 @@ cvmfs_repo_meta: organisation: "{{ cvmfs_server_meta_organisation | default('Your Organisation') }}" description: "Repository content" url: "{{ cvmfs_server_meta_website | default('Project website') }}" - recommended-stratum0: "stratum 0 url" + recommended-stratum0: "stratum 0 url" recommended-stratum1s: ["stratum1 url", "stratum1 url"] custom: "{{ cvmfs_server_meta_custom | default({'_comment': 'Put arbitrary structured data here'}) }}" From ab8b65d112c48793c0d13908fee62a878e750325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 10 Oct 2023 14:18:50 +0200 Subject: [PATCH 12/13] add space --- roles/cvmfs_server_meta_info/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/cvmfs_server_meta_info/tasks/main.yml b/roles/cvmfs_server_meta_info/tasks/main.yml index ee5bed33..53a1a55a 100644 --- a/roles/cvmfs_server_meta_info/tasks/main.yml +++ b/roles/cvmfs_server_meta_info/tasks/main.yml @@ -16,7 +16,7 @@ description: "{{ item.description | default(omit) }}" recommended-stratum0: "http://{{ item.stratum0 }}/cvmfs/{{ item.repository }}" recommended-stratum1s: "{{ eessi_cvmfs_server_urls | selectattr('domain', 'in', item.repository) | - map(attribute='urls') | map('regex_replace', '@fqrn@', item.repository)}}" + map(attribute='urls') | map('regex_replace', '@fqrn@', item.repository) }}" with_items: "{{ eessi_cvmfs_repositories }}" when: "'cvmfsstratum0servers' in group_names or cvmfs_role == 'stratum0'" ... From 9ed0f751bc77ba5f2cbcf8e121f7121a976b8f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 6 Nov 2023 11:25:08 +0100 Subject: [PATCH 13/13] fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Röblitz --- inventory/group_vars/all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 2fcae6ea..164af0e8 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -108,7 +108,7 @@ eessi_cvmfs_repositories: - CVMFS_GARBAGE_COLLECTION=true client_options: [] use_for_ci: true - description: "Small-sized epository used for CI purposes." + description: "Small-sized repository used for CI purposes." # Meta information for CVMFS Stratum servers cvmfs_server_meta_administrator: "EESSI CVMFS Administrators"