From 44c11ffbf9c4cf20b210a63626403e65fb3fcb49 Mon Sep 17 00:00:00 2001 From: David Danielsson Date: Thu, 30 May 2024 08:43:17 -0500 Subject: [PATCH] Offline sync (#387) * init commit * modified: README.md modified: defaults/main.yml modified: meta/argument_specs.yml modified: meta/main.yml modified: tasks/main.yml modified: tests/test.yml * modified: roles/offline_sync/tests/test.yml deleted: roles/offline_sync/tests/vars/collections.yml * modified: roles/offline_sync/templates/collections_list.yml.j2 * new file: changelogs/fragments/offline_sync.yml * modified: roles/collection/README.md modified: roles/offline_sync/README.md modified: roles/offline_sync/tasks/main.yml modified: roles/offline_sync/templates/ansible.cfg.j2 --------- Co-authored-by: Tom Page --- changelogs/fragments/offline_sync.yml | 4 + roles/offline_sync/README.md | 101 ++++++++++++++++++ roles/offline_sync/defaults/main.yml | 13 +++ roles/offline_sync/meta/argument_specs.yml | 47 ++++++++ roles/offline_sync/meta/main.yml | 41 +++++++ roles/offline_sync/tasks/main.yml | 80 ++++++++++++++ roles/offline_sync/templates/ansible.cfg.j2 | 19 ++++ .../offline_sync/templates/collections.yml.j2 | 6 ++ .../templates/collections_list.yml.j2 | 14 +++ roles/offline_sync/tests/test.yml | 13 +++ 10 files changed, 338 insertions(+) create mode 100644 changelogs/fragments/offline_sync.yml create mode 100644 roles/offline_sync/README.md create mode 100644 roles/offline_sync/defaults/main.yml create mode 100644 roles/offline_sync/meta/argument_specs.yml create mode 100644 roles/offline_sync/meta/main.yml create mode 100644 roles/offline_sync/tasks/main.yml create mode 100644 roles/offline_sync/templates/ansible.cfg.j2 create mode 100644 roles/offline_sync/templates/collections.yml.j2 create mode 100644 roles/offline_sync/templates/collections_list.yml.j2 create mode 100644 roles/offline_sync/tests/test.yml diff --git a/changelogs/fragments/offline_sync.yml b/changelogs/fragments/offline_sync.yml new file mode 100644 index 00000000..7c242c7d --- /dev/null +++ b/changelogs/fragments/offline_sync.yml @@ -0,0 +1,4 @@ +--- +minor_changes: + - added offline_sync role +... diff --git a/roles/offline_sync/README.md b/roles/offline_sync/README.md new file mode 100644 index 00000000..0875626c --- /dev/null +++ b/roles/offline_sync/README.md @@ -0,0 +1,101 @@ +# galaxy.galaxy.offline_sync + +## Description + +An Ansible Role to offline_sync collections to Automation Hub or Galaxies. NOTE: if you do not provide an ah_token one will be generated which will invalidate any prior token. + +## Variables + +|Variable Name|Default Value|Required|Description|Example| +|:---:|:---:|:---:|:---:|:---:| +|`ah_host`|""|yes|URL to the Automation Hub or Galaxy Server. (alias: `ah_hostname`)|127.0.0.1| +|`ah_username`|""|yes|Admin User on the Automation Hub or Galaxy Server.|| +|`ah_password`|""|yes|Automation Hub Admin User's password on the Automation Hub Server. This should be stored in an Ansible Vault at vars/tower-secrets.yml or elsewhere and called from a parent playbook.|| +|`ah_token`|""|no|Admin User's token on the Automation Hub Server. This should be stored in an Ansible Vault at or elsewhere and called from a parent playbook.|| +|`ah_validate_certs`|`true`|no|Whether or not to validate the Ansible Automation Hub Server's SSL certificate.|| +|`ah_request_timeout`|`10`|no|Specify the timeout Ansible should use in requests to the Galaxy or Automation Hub host.|| +|`ah_path_prefix`|""|no|API path used to access the api. Either galaxy, automation-hub, or custom|| + +### Asynchronous Retry Variables + +The following Variables set asynchronous retries for the role. +If neither of the retries or delay or retries are set, they will default to their respective defaults. +This allows for all items to be created, then checked that the task finishes successfully. +This also speeds up the overall role. + +|Variable Name|Default Value|Required|Description| +|:---:|:---:|:---:|:---:| +|`ah_configuration_async_retries`|50|no|This variable sets the number of retries to attempt for the role globally.| +|`ah_configuration_collection_async_retries`|`ah_configuration_async_retries`|no|This variable sets the number of retries to attempt for the role.| +|`ah_configuration_async_delay`|1|no|This sets the delay between retries for the role globally.| +|`ah_configuration_collection_async_delay`|`ah_configuration_async_delay`|no|This sets the delay between retries for the role.| + +## Data Structure + +### ah_collections Variables + +|Variable Name|Default Value|Required|Type|Description| +|:---:|:---:|:---:|:---:|:---:| +|`ah_configuration_working_dir`|`/var/tmp/pah_offline_sync`|no|string|The working directory where the collections will be downloaded and any required files.| +|`ah_configuration_no_deps`|false|no|bool|Whether to download all dependencies for each collection or not, if false it may cause errors if dependency sync is off in Automation Hub.| + +## Playbook Examples + +### Standard Role Usage + +```yaml +--- +- name: Download all collections from Automation Hub + hosts: localhost + connection: local + gather_facts: false + vars: + ah_validate_certs: false + # Define following vars here, or in ah_configs/ah_auth.yml + # ah_host: ansible-ah-web-svc-test-project.example.com + # ah_token: changeme + pre_tasks: + - name: Include vars from ah_configs directory + ansible.builtin.include_vars: + dir: ./vars + extensions: ["yml"] + tags: + - always + roles: + - galaxy.galaxy.offline_sync +``` + +### Playbook to upload to offline Automation Hub after using this role to download the collections. + +```yaml +--- +- name: Upload all collections + hosts: localhost + gather_facts: false + connection: local + vars_files: + - "collections.yml" + pre_tasks: + - name: Include vars from ah_configs directory with collections.yml file added + ansible.builtin.include_vars: + dir: ./vars + extensions: ["yml"] + tags: + - always + tasks: + - name: Ensure the namespaces exists + ansible.builtin.import_role: + name: galaxy.galaxy.namespace + + - name: Upload collections + ansible.builtin.include_role: + name: galaxy.galaxy.collection +``` + +## License + +[GPLv3+](https://github.com/ansible/galaxy_collection#licensing) + +## Author + +[David Danielsson](https://github.com/djdanielsson) diff --git a/roles/offline_sync/defaults/main.yml b/roles/offline_sync/defaults/main.yml new file mode 100644 index 00000000..c3bcf2e8 --- /dev/null +++ b/roles/offline_sync/defaults/main.yml @@ -0,0 +1,13 @@ +--- + +# These are the default variables common to most ah_configuration and _utilities roles +# You shouldn't need to define them again and again but they should be defined +# ah_hostname: "{{ inventory_hostname }}" +# ah_oauthtoken: "" +# ah_validate_certs: false + +ah_configuration_working_dir: /var/tmp/pah_offline_sync +ah_configuration_no_deps: false +# ah_overwrite_existing: false + +... diff --git a/roles/offline_sync/meta/argument_specs.yml b/roles/offline_sync/meta/argument_specs.yml new file mode 100644 index 00000000..4104f6d1 --- /dev/null +++ b/roles/offline_sync/meta/argument_specs.yml @@ -0,0 +1,47 @@ +--- +argument_specs: + main: + short_description: An Ansible Role to offline sync collections in Automation Hub. + options: + ah_configuration_working_dir: + default: /var/tmp/pah_offline_sync + type: str + required: false + description: The working directory where the collections will be downloaded and any required files. + ah_configuration_no_deps: + default: false + type: bool + required: false + description: Whether to download all dependencies for each collection or not, if false it may cause errors if dependency sync is off in Automation Hub. + + # Generic across all roles + ah_host: + required: false + description: URL to the Automation Hub Server. + type: str + ah_path_prefix: + required: false + description: The path for the Automation Hub API. Usually galaxy or automation-hub unless custom set in AH settings. + ah_validate_certs: + required: false + description: Whether or not to validate the Automation Hub Server's SSL certificate. + type: str + ah_request_timeout: + default: 10 + required: false + description: Specify the timeout Ansible should use in requests to the Galaxy or Automation Hub host. + type: float + ah_username: + required: false + description: User for authentication on Automation Hub + type: str + ah_password: + required: false + description: User's password For Automation Hub + type: str + ah_token: + required: false + description: User's token on the Automation Hub Server + type: str + +... diff --git a/roles/offline_sync/meta/main.yml b/roles/offline_sync/meta/main.yml new file mode 100644 index 00000000..ab626ce4 --- /dev/null +++ b/roles/offline_sync/meta/main.yml @@ -0,0 +1,41 @@ +--- +galaxy_info: + role_name: offline_sync + author: David Danielsson + description: An Ansible Role to offline_sync collections to Automation Hub or Galaxies. + company: Red Hat + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + license: GPL-3.0-or-later + + min_ansible_version: 2.14.0 + + # Optionally specify the branch Galaxy will use when accessing the GitHub + # repo for this role. During role install, if no tags are available, + # Galaxy will use this branch. During import Galaxy will access files on + # this branch. If Travis integration is configured, only notifications for this + # branch will be accepted. Otherwise, in all cases, the repo's default branch + # (usually master) will be used. + + # github_branch: + + # + # platforms is a list of platforms, and each platform has a name and a list of versions. + # + platforms: + - name: EL + versions: + - all + + galaxy_tags: + - automationhub + - galaxy + - configuration + - collection + - collections + - offlinesync + +dependencies: [] +... diff --git a/roles/offline_sync/tasks/main.yml b/roles/offline_sync/tasks/main.yml new file mode 100644 index 00000000..26ae74f0 --- /dev/null +++ b/roles/offline_sync/tasks/main.yml @@ -0,0 +1,80 @@ +--- +- name: Create directory to work in + ansible.builtin.file: + path: "{{ ah_configuration_working_dir }}" + state: directory + mode: "0755" + +- name: Check for ansible.cfg + ansible.builtin.stat: + path: "{{ ah_configuration_working_dir }}/ansible.cfg" + register: r_ansible_cfg + +- name: Figuring out AH token + when: (not r_ansible_cfg['stat']['exists']) and (ah_token is not defined or ah_token['token'] is defined) + block: + - name: Authenticate and get an API token from Automation Hub + ah_token: + ah_host: "{{ ah_host }}" + ah_username: "{{ ah_username }}" + ah_password: "{{ ah_password }}" + ah_path_prefix: galaxy # this is for private automation hub + validate_certs: false + register: r_ah_token + + - name: Fixing format + ansible.builtin.set_fact: + ah_token: "{{ ah_token['token'] }}" + when: r_ah_token['changed'] # noqa: no-handler + +- name: Create ansible.cfg + ansible.builtin.template: + src: ansible.cfg.j2 + dest: "{{ ah_configuration_working_dir }}/ansible.cfg" + mode: "0755" + when: not r_ansible_cfg['stat']['exists'] + +- name: Get list of all collections + ansible.builtin.uri: + url: https://{{ ah_host }}/api/galaxy/pulp/api/v3/ansible/collections/?limit=1000 + user: "{{ ah_username }}" + password: "{{ ah_password }}" + method: GET + force_basic_auth: true + validate_certs: "{{ ah_validate_certs }}" + register: r_collections + +- name: Create requirements.yml + ansible.builtin.template: + src: collections.yml.j2 + dest: "{{ ah_configuration_working_dir }}/requirements.yml" + mode: "0755" + +- name: Create directory to put all collections + ansible.builtin.file: + path: "{{ ah_configuration_working_dir }}/collections" + state: directory + mode: "0755" + +- name: Download all collections + ansible.builtin.command: ansible-galaxy collection download {% if ah_configuration_no_deps %}--no-deps{% endif %} -r requirements.yml -p "{{ ah_configuration_working_dir }}/collections" + changed_when: true + +- name: Find all relevant built collection files + ansible.builtin.find: + paths: "{{ collection_dir }}" + patterns: "{{ (collection_dir) | ternary('*.tar.gz', (collection_dir | basename)) }}" + register: __ah_collections_find_results + vars: + collection_dir: "{{ ah_configuration_working_dir }}/collections/" + +- name: Create collections list + ansible.builtin.template: + src: collections_list.yml.j2 + dest: "{{ ah_configuration_working_dir }}/collections.yml" + mode: "0755" + vars: + _namespaces: "{{ r_collections['json']['results'] | map(attribute='namespace') | flatten | sort | unique }}" + _collections: "{{ (__ah_collections_find_results['files'] | flatten) | map(attribute='path') | flatten | sort | unique }}" + +... diff --git a/roles/offline_sync/templates/ansible.cfg.j2 b/roles/offline_sync/templates/ansible.cfg.j2 new file mode 100644 index 00000000..809d667c --- /dev/null +++ b/roles/offline_sync/templates/ansible.cfg.j2 @@ -0,0 +1,19 @@ +[galaxy] +server_list = rh-certified,validated,community,published +ignore_certs = {% if ah_validate_certs %}false{% else %}true{% endif %} + +[galaxy_server.rh-certified] +url=https://{{ ah_host }}/api/galaxy/content/rh-certified/ +token={{ ah_token }} + +[galaxy_server.validated] +url=https://{{ ah_host }}/api/galaxy/content/validated/ +token={{ ah_token }} + +[galaxy_server.community] +url=https://{{ ah_host }}/api/galaxy/content/community/ +token={{ ah_token }} + +[galaxy_server.published] +url=https://{{ ah_host }}/api/galaxy/content/published/ +token={{ ah_token }} diff --git a/roles/offline_sync/templates/collections.yml.j2 b/roles/offline_sync/templates/collections.yml.j2 new file mode 100644 index 00000000..bbf10810 --- /dev/null +++ b/roles/offline_sync/templates/collections.yml.j2 @@ -0,0 +1,6 @@ +--- +collections: +{% for each in r_collections['json']['results'] %} + - name: {{ each.namespace }}.{{ each.name }} +{% endfor %} +... diff --git a/roles/offline_sync/templates/collections_list.yml.j2 b/roles/offline_sync/templates/collections_list.yml.j2 new file mode 100644 index 00000000..2f5168cc --- /dev/null +++ b/roles/offline_sync/templates/collections_list.yml.j2 @@ -0,0 +1,14 @@ +--- +ah_auto_approve: true +ah_namespaces: +{% for namespace in _namespaces %} + - name: {{ namespace }} +{% endfor %} +ah_collections: +{% for each in _collections %} + - name: {{ (each | basename).split('-')[1] }} + namespace: {{ (each | basename).split('-')[0] }} + path: {{ each }} + state: present +{% endfor %} +... diff --git a/roles/offline_sync/tests/test.yml b/roles/offline_sync/tests/test.yml new file mode 100644 index 00000000..e442a2a3 --- /dev/null +++ b/roles/offline_sync/tests/test.yml @@ -0,0 +1,13 @@ +--- +- name: Offline_sync collections to Automation Hub + hosts: localhost + connection: local + gather_facts: false + vars: + ah_validate_certs: false + # Define following vars here, or in ah_configs/ah_auth.yml + # ah_host: ansible-ah-web-svc-test-project.example.com + # ah_token: changeme + roles: + - ../../offline_sync +...