Offline sync (#387)

* init commit

* modified:   README.md
	modified:   defaults/main.yml
	modified:   meta/argument_specs.yml
	modified:   meta/main.yml
	modified:   tasks/main.yml
	modified:   tests/test.yml

* modified:   roles/offline_sync/tests/test.yml
	deleted:    roles/offline_sync/tests/vars/collections.yml

* modified:   roles/offline_sync/templates/collections_list.yml.j2

* new file:   changelogs/fragments/offline_sync.yml

* modified:   roles/collection/README.md
	modified:   roles/offline_sync/README.md
	modified:   roles/offline_sync/tasks/main.yml
	modified:   roles/offline_sync/templates/ansible.cfg.j2

---------

Co-authored-by: Tom Page <tpage@redhat.com>

changelogs/fragments/offline_sync.yml

@@ -0,0 +1,4 @@
+---
+minor_changes:
+  - added offline_sync role
+...

roles/offline_sync/README.md

@@ -0,0 +1,101 @@
+# galaxy.galaxy.offline_sync
+
+## Description
+
+An Ansible Role to offline_sync collections to Automation Hub or Galaxies. NOTE: if you do not provide an ah_token one will be generated which will invalidate any prior token.
+
+## Variables
+
+|Variable Name|Default Value|Required|Description|Example|
+|:---:|:---:|:---:|:---:|:---:|
+|`ah_host`|""|yes|URL to the Automation Hub or Galaxy Server. (alias: `ah_hostname`)|127.0.0.1|
+|`ah_username`|""|yes|Admin User on the Automation Hub or Galaxy Server.||
+|`ah_password`|""|yes|Automation Hub Admin User's password on the Automation Hub Server. This should be stored in an Ansible Vault at vars/tower-secrets.yml or elsewhere and called from a parent playbook.||
+|`ah_token`|""|no|Admin User's token on the Automation Hub Server. This should be stored in an Ansible Vault at or elsewhere and called from a parent playbook.||
+|`ah_validate_certs`|`true`|no|Whether or not to validate the Ansible Automation Hub Server's SSL certificate.||
+|`ah_request_timeout`|`10`|no|Specify the timeout Ansible should use in requests to the Galaxy or Automation Hub host.||
+|`ah_path_prefix`|""|no|API path used to access the api. Either galaxy, automation-hub, or custom||
+
+### Asynchronous Retry Variables
+
+The following Variables set asynchronous retries for the role.
+If neither of the retries or delay or retries are set, they will default to their respective defaults.
+This allows for all items to be created, then checked that the task finishes successfully.
+This also speeds up the overall role.
+
+|Variable Name|Default Value|Required|Description|
+|:---:|:---:|:---:|:---:|
+|`ah_configuration_async_retries`|50|no|This variable sets the number of retries to attempt for the role globally.|
+|`ah_configuration_collection_async_retries`|`ah_configuration_async_retries`|no|This variable sets the number of retries to attempt for the role.|
+|`ah_configuration_async_delay`|1|no|This sets the delay between retries for the role globally.|
+|`ah_configuration_collection_async_delay`|`ah_configuration_async_delay`|no|This sets the delay between retries for the role.|
+
+## Data Structure
+
+### ah_collections Variables
+
+|Variable Name|Default Value|Required|Type|Description|
+|:---:|:---:|:---:|:---:|:---:|
+|`ah_configuration_working_dir`|`/var/tmp/pah_offline_sync`|no|string|The working directory where the collections will be downloaded and any required files.|
+|`ah_configuration_no_deps`|false|no|bool|Whether to download all dependencies for each collection or not, if false it may cause errors if dependency sync is off in Automation Hub.|
+
+## Playbook Examples
+
+### Standard Role Usage
+
+```yaml
+---
+- name: Download all collections from Automation Hub
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  vars:
+    ah_validate_certs: false
+  # Define following vars here, or in ah_configs/ah_auth.yml
+  # ah_host: ansible-ah-web-svc-test-project.example.com
+  # ah_token: changeme
+  pre_tasks:
+    - name: Include vars from ah_configs directory
+      ansible.builtin.include_vars:
+        dir: ./vars
+        extensions: ["yml"]
+      tags:
+        - always
+  roles:
+    - galaxy.galaxy.offline_sync
+```
+
+### Playbook to upload to offline Automation Hub after using this role to download the collections.
+
+```yaml
+---
+- name: Upload all collections
+  hosts: localhost
+  gather_facts: false
+  connection: local
+  vars_files:
+    - "collections.yml"
+  pre_tasks:
+    - name: Include vars from ah_configs directory with collections.yml file added
+      ansible.builtin.include_vars:
+        dir: ./vars
+        extensions: ["yml"]
+      tags:
+        - always
+  tasks:
+    - name: Ensure the namespaces exists
+      ansible.builtin.import_role:
+        name: galaxy.galaxy.namespace
+
+    - name: Upload collections
+      ansible.builtin.include_role:
+        name: galaxy.galaxy.collection
+```
+
+## License
+
+[GPLv3+](https://github.com/ansible/galaxy_collection#licensing)
+
+## Author
+
+[David Danielsson](https://github.com/djdanielsson)

roles/offline_sync/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+
+# These are the default variables common to most ah_configuration and _utilities roles
+# You shouldn't need to define them again and again but they should be defined
+# ah_hostname: "{{ inventory_hostname }}"
+# ah_oauthtoken: ""
+# ah_validate_certs: false
+
+ah_configuration_working_dir: /var/tmp/pah_offline_sync
+ah_configuration_no_deps: false
+# ah_overwrite_existing: false
+
+...

roles/offline_sync/meta/argument_specs.yml

@@ -0,0 +1,47 @@
+---
+argument_specs:
+  main:
+    short_description: An Ansible Role to offline sync collections in Automation Hub.
+    options:
+      ah_configuration_working_dir:
+        default: /var/tmp/pah_offline_sync
+        type: str
+        required: false
+        description: The working directory where the collections will be downloaded and any required files.
+      ah_configuration_no_deps:
+        default: false
+        type: bool
+        required: false
+        description: Whether to download all dependencies for each collection or not, if false it may cause errors if dependency sync is off in Automation Hub.
+
+      # Generic across all roles
+      ah_host:
+        required: false
+        description: URL to the Automation Hub Server.
+        type: str
+      ah_path_prefix:
+        required: false
+        description: The path for the Automation Hub API. Usually galaxy or automation-hub unless custom set in AH settings.
+      ah_validate_certs:
+        required: false
+        description: Whether or not to validate the Automation Hub Server's SSL certificate.
+        type: str
+      ah_request_timeout:
+        default: 10
+        required: false
+        description: Specify the timeout Ansible should use in requests to the Galaxy or Automation Hub host.
+        type: float
+      ah_username:
+        required: false
+        description: User for authentication on Automation Hub
+        type: str
+      ah_password:
+        required: false
+        description: User's password For Automation Hub
+        type: str
+      ah_token:
+        required: false
+        description: User's token on the Automation Hub Server
+        type: str
+
+...

roles/offline_sync/meta/main.yml

@@ -0,0 +1,41 @@
+---
+galaxy_info:
+  role_name: offline_sync
+  author: David Danielsson
+  description: An Ansible Role to offline_sync collections to Automation Hub or Galaxies.
+  company: Red Hat
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+  license: GPL-3.0-or-later
+
+  min_ansible_version: 2.14.0
+
+  # Optionally specify the branch Galaxy will use when accessing the GitHub
+  # repo for this role. During role install, if no tags are available,
+  # Galaxy will use this branch. During import Galaxy will access files on
+  # this branch. If Travis integration is configured, only notifications for this
+  # branch will be accepted. Otherwise, in all cases, the repo's default branch
+  # (usually master) will be used.
+
+  # github_branch:
+
+  #
+  # platforms is a list of platforms, and each platform has a name and a list of versions.
+  #
+  platforms:
+    - name: EL
+      versions:
+        - all
+
+  galaxy_tags:
+    - automationhub
+    - galaxy
+    - configuration
+    - collection
+    - collections
+    - offlinesync
+
+dependencies: []
+...

roles/offline_sync/tasks/main.yml

@@ -0,0 +1,80 @@
+---
+- name: Create directory to work in
+  ansible.builtin.file:
+    path: "{{ ah_configuration_working_dir }}"
+    state: directory
+    mode: "0755"
+
+- name: Check for ansible.cfg
+  ansible.builtin.stat:
+    path: "{{ ah_configuration_working_dir }}/ansible.cfg"
+  register: r_ansible_cfg
+
+- name: Figuring out AH token
+  when: (not r_ansible_cfg['stat']['exists']) and (ah_token is not defined or ah_token['token'] is defined)
+  block:
+    - name: Authenticate and get an API token from Automation Hub
+      ah_token:
+        ah_host: "{{ ah_host }}"
+        ah_username: "{{ ah_username }}"
+        ah_password: "{{ ah_password }}"
+        ah_path_prefix: galaxy  # this is for private automation hub
+        validate_certs: false
+      register: r_ah_token
+
+    - name: Fixing format
+      ansible.builtin.set_fact:
+        ah_token: "{{ ah_token['token'] }}"
+      when: r_ah_token['changed']  # noqa: no-handler
+
+- name: Create ansible.cfg
+  ansible.builtin.template:
+    src: ansible.cfg.j2
+    dest: "{{ ah_configuration_working_dir }}/ansible.cfg"
+    mode: "0755"
+  when: not r_ansible_cfg['stat']['exists']
+
+- name: Get list of all collections
+  ansible.builtin.uri:
+    url: https://{{ ah_host }}/api/galaxy/pulp/api/v3/ansible/collections/?limit=1000
+    user: "{{ ah_username }}"
+    password: "{{ ah_password }}"
+    method: GET
+    force_basic_auth: true
+    validate_certs: "{{ ah_validate_certs }}"
+  register: r_collections
+
+- name: Create requirements.yml
+  ansible.builtin.template:
+    src: collections.yml.j2
+    dest: "{{ ah_configuration_working_dir }}/requirements.yml"
+    mode: "0755"
+
+- name: Create directory to put all collections
+  ansible.builtin.file:
+    path: "{{ ah_configuration_working_dir }}/collections"
+    state: directory
+    mode: "0755"
+
+- name: Download all collections
+  ansible.builtin.command: ansible-galaxy collection download {% if ah_configuration_no_deps %}--no-deps{% endif %} -r requirements.yml -p "{{ ah_configuration_working_dir }}/collections"
+  changed_when: true
+
+- name: Find all relevant built collection files
+  ansible.builtin.find:
+    paths: "{{ collection_dir }}"
+    patterns: "{{ (collection_dir) | ternary('*.tar.gz', (collection_dir | basename)) }}"
+  register: __ah_collections_find_results
+  vars:
+    collection_dir: "{{ ah_configuration_working_dir }}/collections/"
+
+- name: Create collections list
+  ansible.builtin.template:
+    src: collections_list.yml.j2
+    dest: "{{ ah_configuration_working_dir }}/collections.yml"
+    mode: "0755"
+  vars:
+    _namespaces: "{{ r_collections['json']['results'] | map(attribute='namespace') | flatten | sort | unique }}"
+    _collections: "{{ (__ah_collections_find_results['files'] | flatten) | map(attribute='path') | flatten | sort | unique }}"
+
+...

roles/offline_sync/templates/ansible.cfg.j2

@@ -0,0 +1,19 @@
+[galaxy]
+server_list = rh-certified,validated,community,published
+ignore_certs = {% if ah_validate_certs %}false{% else %}true{% endif %}
+
+[galaxy_server.rh-certified]
+url=https://{{ ah_host }}/api/galaxy/content/rh-certified/
+token={{ ah_token }}
+
+[galaxy_server.validated]
+url=https://{{ ah_host }}/api/galaxy/content/validated/
+token={{ ah_token }}
+
+[galaxy_server.community]
+url=https://{{ ah_host }}/api/galaxy/content/community/
+token={{ ah_token }}
+
+[galaxy_server.published]
+url=https://{{ ah_host }}/api/galaxy/content/published/
+token={{ ah_token }}

roles/offline_sync/templates/collections.yml.j2

@@ -0,0 +1,6 @@
+---
+collections:
+{% for each in r_collections['json']['results'] %}
+  - name: {{ each.namespace }}.{{ each.name }}
+{% endfor %}
+...

roles/offline_sync/templates/collections_list.yml.j2

@@ -0,0 +1,14 @@
+---
+ah_auto_approve: true
+ah_namespaces:
+{% for namespace in _namespaces %}
+  - name: {{ namespace }}
+{% endfor %}
+ah_collections:
+{% for each in _collections %}
+  - name: {{ (each | basename).split('-')[1] }}
+    namespace: {{ (each | basename).split('-')[0] }}
+    path: {{ each }}
+    state: present
+{% endfor %}
+...

roles/offline_sync/tests/test.yml

@@ -0,0 +1,13 @@
+---
+- name: Offline_sync collections to Automation Hub
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  vars:
+    ah_validate_certs: false
+  # Define following vars here, or in ah_configs/ah_auth.yml
+  # ah_host: ansible-ah-web-svc-test-project.example.com
+  # ah_token: changeme
+  roles:
+    - ../../offline_sync
+...