From 13c505a3ab91c7abcd3ea7c5b765e01a57d3b43c Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 23 Sep 2024 11:30:14 +0000 Subject: [PATCH 01/10] Install WAL-G rom the binary file --- automation/roles/wal-g/tasks/main.yml | 56 ++++++++++++++++++++++----- automation/vars/main.yml | 1 + 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/automation/roles/wal-g/tasks/main.yml b/automation/roles/wal-g/tasks/main.yml index 1326ec97b..6ab96ab91 100644 --- a/automation/roles/wal-g/tasks/main.yml +++ b/automation/roles/wal-g/tasks/main.yml @@ -8,6 +8,7 @@ - wal_g_auto_conf | default(true) | bool # to be able to disable auto backup settings tags: wal-g, wal_g, wal_g_conf +# Pre-check - name: Check if WAL-G is already installed ansible.builtin.shell: | set -o pipefail; @@ -29,7 +30,43 @@ - wal_g_installed_version.stdout == wal_g_version tags: wal-g, wal_g, wal_g_install -# Build WAL-G from source code for other Linux distributions +# Install WAL-G from a precompiled binary +# (if 'wal_g_installation_method' is 'binary') +- block: + - name: "Download WAL-G v{{ wal_g_version }} binary" + ansible.builtin.get_url: + url: "{{ wal_g_repo }}/{{ wal_g_archive }}" + dest: /tmp/ + timeout: 60 + validate_certs: false + vars: + wal_g_repo: "https://github.com/wal-g/wal-g/releases/download/v{{ wal_g_version }}" + wal_g_archive: "wal-g-pg-ubuntu-20.04-amd64.tar.gz" + environment: "{{ proxy_env | default({}) }}" + + - name: Extract WAL-G into /tmp + ansible.builtin.unarchive: + src: "/tmp/wal-g-pg-ubuntu-20.04-amd64.tar.gz" + dest: /tmp/ + extra_opts: + - --no-same-owner + remote_src: true + + - name: Copy WAL-G binary file to "{{ wal_g_path | basename }}" + ansible.builtin.copy: + src: "/tmp/wal-g-pg-ubuntu-20.04-amd64" + dest: "{{ wal_g_path | basename }}" + mode: u+x,g+x,o+x + remote_src: true + when: + - installation_method == "repo" + - wal_g_installation_method == "binary" + - wal_g_version is version('1.0', '>=') + - (wal_g_installed_version.stderr is search("command not found") or wal_g_installed_version.stdout != wal_g_version) + tags: wal-g, wal_g, wal_g_install + +# Install WAL-G from the source code +# (if 'wal_g_installation_method' is 'src') - block: - name: Install lib dependencies to build WAL-G ansible.builtin.package: @@ -145,6 +182,7 @@ environment: "{{ proxy_env | default({}) }}" when: - installation_method == "repo" + - wal_g_installation_method == "src" - wal_g_version is version('1.0', '>=') - (wal_g_installed_version.stderr is search("command not found") or wal_g_installed_version.stdout != wal_g_version) tags: wal-g, wal_g, wal_g_install @@ -167,17 +205,17 @@ - --no-same-owner remote_src: true - - name: Copy WAL-G binary file to "{{ wal_g_path }}" + - name: Copy WAL-G binary file to "{{ wal_g_path | basename }}" ansible.builtin.copy: src: "/tmp/wal-g" - dest: "{{ wal_g_path }}" + dest: "{{ wal_g_path | basename }}" mode: u+x,g+x,o+x remote_src: true when: - installation_method == "repo" + - wal_g_installation_method == "binary" - wal_g_version is version('0.2.19', '<=') - - (wal_g_installed_version.stderr is search("command not found") or - wal_g_installed_version.stdout != wal_g_version) + - (wal_g_installed_version.stderr is search("command not found") or wal_g_installed_version.stdout != wal_g_version) tags: wal-g, wal_g, wal_g_install # installation_method == "file" @@ -191,10 +229,10 @@ extra_opts: - --no-same-owner - - name: Copy WAL-G binary file to "{{ wal_g_path }}" + - name: Copy WAL-G binary file to "{{ wal_g_path | basename }}" ansible.builtin.copy: src: "/tmp/{{ wal_g_package_file.split('.tar.gz')[0] | basename }}" - dest: "{{ wal_g_path }}" + dest: "{{ wal_g_path | basename }}" mode: u+x,g+x,o+x remote_src: true when: @@ -214,10 +252,10 @@ extra_opts: - --no-same-owner - - name: Copy WAL-G binary file to "{{ wal_g_path }}" + - name: Copy WAL-G binary file to "{{ wal_g_path | basename }}" ansible.builtin.copy: src: "/tmp/wal-g" - dest: "{{ wal_g_path }}" + dest: "{{ wal_g_path | basename }}" mode: u+x,g+x,o+x remote_src: true when: diff --git a/automation/vars/main.yml b/automation/vars/main.yml index 267a88229..9646c72ca 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -504,6 +504,7 @@ pg_probackup_patroni_cluster_bootstrap_command: "{{ pg_probackup_command_parts | # WAL-G wal_g_install: false # or 'true' wal_g_version: "3.0.3" +wal_g_installation_method: "binary" # or "src" to build from source code wal_g_path: "/usr/local/bin/wal-g --config {{ postgresql_home_dir }}/.walg.json" wal_g_json: # config https://github.com/wal-g/wal-g#configuration - { option: "AWS_ACCESS_KEY_ID", value: "{{ AWS_ACCESS_KEY_ID | default('') }}" } # define values or pass via --extra-vars From 9a0c96d61e44d65af25f81859045a5a7fa9990e9 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 23 Sep 2024 11:36:29 +0000 Subject: [PATCH 02/10] Add a note about using the binary --- automation/roles/wal-g/tasks/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/automation/roles/wal-g/tasks/main.yml b/automation/roles/wal-g/tasks/main.yml index 6ab96ab91..2f902101e 100644 --- a/automation/roles/wal-g/tasks/main.yml +++ b/automation/roles/wal-g/tasks/main.yml @@ -44,6 +44,9 @@ wal_g_archive: "wal-g-pg-ubuntu-20.04-amd64.tar.gz" environment: "{{ proxy_env | default({}) }}" + # Note: We are using a precompiled binary on Ubuntu 20.04, + # but since Go binaries are cross-platform, it works well on other distributions as well. + - name: Extract WAL-G into /tmp ansible.builtin.unarchive: src: "/tmp/wal-g-pg-ubuntu-20.04-amd64.tar.gz" From 162b2319e8c8476e70171af24cdaac0855004e53 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 23 Sep 2024 11:41:17 +0000 Subject: [PATCH 03/10] wal_g_install: true (for test) --- automation/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/vars/main.yml b/automation/vars/main.yml index 9646c72ca..9ec6f289f 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -502,7 +502,7 @@ pg_probackup_restore_command: "{{ pg_probackup_command_parts | join('') }}" pg_probackup_patroni_cluster_bootstrap_command: "{{ pg_probackup_command_parts | join('') }}" # WAL-G -wal_g_install: false # or 'true' +wal_g_install: true # or 'true' wal_g_version: "3.0.3" wal_g_installation_method: "binary" # or "src" to build from source code wal_g_path: "/usr/local/bin/wal-g --config {{ postgresql_home_dir }}/.walg.json" From c46bbf4ddfd3aaf4fb736c9882e79bb198969c63 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 23 Sep 2024 11:47:36 +0000 Subject: [PATCH 04/10] Add force option for git --- automation/roles/wal-g/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/automation/roles/wal-g/tasks/main.yml b/automation/roles/wal-g/tasks/main.yml index 2f902101e..1b8ad400f 100644 --- a/automation/roles/wal-g/tasks/main.yml +++ b/automation/roles/wal-g/tasks/main.yml @@ -148,6 +148,7 @@ repo: https://github.com/wal-g/wal-g.git version: v{{ wal_g_version }} dest: /tmp/wal-g + force: true - name: Run go mod tidy to ensure dependencies are correct ansible.builtin.command: go mod tidy From 36bb8c61a43b0bfb39fddb2bd7c6d8568763d210 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 23 Sep 2024 11:51:58 +0000 Subject: [PATCH 05/10] Add replace --- automation/roles/wal-g/tasks/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/automation/roles/wal-g/tasks/main.yml b/automation/roles/wal-g/tasks/main.yml index 1b8ad400f..69c456b22 100644 --- a/automation/roles/wal-g/tasks/main.yml +++ b/automation/roles/wal-g/tasks/main.yml @@ -33,14 +33,14 @@ # Install WAL-G from a precompiled binary # (if 'wal_g_installation_method' is 'binary') - block: - - name: "Download WAL-G v{{ wal_g_version }} binary" + - name: "Download WAL-G v{{ wal_g_version | replace('v', '') }} binary" ansible.builtin.get_url: url: "{{ wal_g_repo }}/{{ wal_g_archive }}" dest: /tmp/ timeout: 60 validate_certs: false vars: - wal_g_repo: "https://github.com/wal-g/wal-g/releases/download/v{{ wal_g_version }}" + wal_g_repo: "https://github.com/wal-g/wal-g/releases/download/v{{ wal_g_version | replace('v', '') }}" wal_g_archive: "wal-g-pg-ubuntu-20.04-amd64.tar.gz" environment: "{{ proxy_env | default({}) }}" @@ -143,10 +143,10 @@ when: go_installed_version.stderr is search("command not found") or go_installed_version.stdout is version(wal_g_latest_version.stdout, '<') - - name: "Download WAL-G v{{ wal_g_version }} source code" + - name: "Download WAL-G v{{ wal_g_version | replace('v', '') }} source code" ansible.builtin.git: repo: https://github.com/wal-g/wal-g.git - version: v{{ wal_g_version }} + version: v{{ wal_g_version | replace('v', '') }} dest: /tmp/wal-g force: true @@ -193,9 +193,9 @@ # older versions of WAL-G (for compatibility) - block: - - name: "Download WAL-G v{{ wal_g_version }} binary" + - name: "Download WAL-G v{{ wal_g_version | replace('v', '') }} binary" ansible.builtin.get_url: - url: "https://github.com/wal-g/wal-g/releases/download/v{{ wal_g_version }}/wal-g.linux-amd64.tar.gz" + url: "https://github.com/wal-g/wal-g/releases/download/v{{ wal_g_version | replace('v', '') }}/wal-g.linux-amd64.tar.gz" dest: /tmp/ timeout: 60 validate_certs: false From 0cb4974bff07dcf580a80087206fb2f50d8a1923 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 23 Sep 2024 12:03:53 +0000 Subject: [PATCH 06/10] Fix wal_g_path --- automation/roles/wal-g/tasks/main.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/automation/roles/wal-g/tasks/main.yml b/automation/roles/wal-g/tasks/main.yml index 69c456b22..0e841db87 100644 --- a/automation/roles/wal-g/tasks/main.yml +++ b/automation/roles/wal-g/tasks/main.yml @@ -12,7 +12,7 @@ - name: Check if WAL-G is already installed ansible.builtin.shell: | set -o pipefail; - "{{ wal_g_path }}" --version | awk {'print $3'} | tr -d 'v' + "{{ wal_g_path.split(' ')[0] }}" --version | awk {'print $3'} | tr -d 'v' args: executable: /bin/bash changed_when: false @@ -55,10 +55,10 @@ - --no-same-owner remote_src: true - - name: Copy WAL-G binary file to "{{ wal_g_path | basename }}" + - name: Copy WAL-G binary file to "{{ wal_g_path.split(' ')[0] }}" ansible.builtin.copy: src: "/tmp/wal-g-pg-ubuntu-20.04-amd64" - dest: "{{ wal_g_path | basename }}" + dest: "{{ wal_g_path.split(' ')[0] }}" mode: u+x,g+x,o+x remote_src: true when: @@ -209,10 +209,10 @@ - --no-same-owner remote_src: true - - name: Copy WAL-G binary file to "{{ wal_g_path | basename }}" + - name: Copy WAL-G binary file to "{{ wal_g_path.split(' ')[0] }}" ansible.builtin.copy: src: "/tmp/wal-g" - dest: "{{ wal_g_path | basename }}" + dest: "{{ wal_g_path.split(' ')[0] }}" mode: u+x,g+x,o+x remote_src: true when: @@ -233,10 +233,10 @@ extra_opts: - --no-same-owner - - name: Copy WAL-G binary file to "{{ wal_g_path | basename }}" + - name: Copy WAL-G binary file to "{{ wal_g_path.split(' ')[0] }}" ansible.builtin.copy: src: "/tmp/{{ wal_g_package_file.split('.tar.gz')[0] | basename }}" - dest: "{{ wal_g_path | basename }}" + dest: "{{ wal_g_path.split(' ')[0] }}" mode: u+x,g+x,o+x remote_src: true when: @@ -256,10 +256,10 @@ extra_opts: - --no-same-owner - - name: Copy WAL-G binary file to "{{ wal_g_path | basename }}" + - name: Copy WAL-G binary file to "{{ wal_g_path.split(' ')[0] }}" ansible.builtin.copy: src: "/tmp/wal-g" - dest: "{{ wal_g_path | basename }}" + dest: "{{ wal_g_path.split(' ')[0] }}" mode: u+x,g+x,o+x remote_src: true when: From 7e3f79634ccfa9c2798883a0d894f2bb7d3a8a45 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 23 Sep 2024 13:36:03 +0000 Subject: [PATCH 07/10] wal_g_install: false (default) --- automation/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/vars/main.yml b/automation/vars/main.yml index 9ec6f289f..9646c72ca 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -502,7 +502,7 @@ pg_probackup_restore_command: "{{ pg_probackup_command_parts | join('') }}" pg_probackup_patroni_cluster_bootstrap_command: "{{ pg_probackup_command_parts | join('') }}" # WAL-G -wal_g_install: true # or 'true' +wal_g_install: false # or 'true' wal_g_version: "3.0.3" wal_g_installation_method: "binary" # or "src" to build from source code wal_g_path: "/usr/local/bin/wal-g --config {{ postgresql_home_dir }}/.walg.json" From 1a6b43e54515f75aed8914e2fa66ab8dea1f203f Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Mon, 30 Sep 2024 15:22:26 +0300 Subject: [PATCH 08/10] exclude RHEL 8 for binory Excluding the RHEL 8, as GLIBC version 2.29 or higher is required. For these versions, there will always be an assembly from the source code, regardless of the value of the wal_g_installation_method variable. --- automation/roles/wal-g/tasks/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/automation/roles/wal-g/tasks/main.yml b/automation/roles/wal-g/tasks/main.yml index 0e841db87..26ce2f4e5 100644 --- a/automation/roles/wal-g/tasks/main.yml +++ b/automation/roles/wal-g/tasks/main.yml @@ -32,6 +32,7 @@ # Install WAL-G from a precompiled binary # (if 'wal_g_installation_method' is 'binary') +# Note: excluding RHEL 8 as GLIBC version 2.29 or higher is required. - block: - name: "Download WAL-G v{{ wal_g_version | replace('v', '') }} binary" ansible.builtin.get_url: @@ -66,6 +67,7 @@ - wal_g_installation_method == "binary" - wal_g_version is version('1.0', '>=') - (wal_g_installed_version.stderr is search("command not found") or wal_g_installed_version.stdout != wal_g_version) + - not (ansible_os_family == "RedHat" and ansible_distribution_major_version == '8') tags: wal-g, wal_g, wal_g_install # Install WAL-G from the source code @@ -186,7 +188,7 @@ environment: "{{ proxy_env | default({}) }}" when: - installation_method == "repo" - - wal_g_installation_method == "src" + - (wal_g_installation_method == "src" or (ansible_os_family == "RedHat" and ansible_distribution_major_version == '8')) - wal_g_version is version('1.0', '>=') - (wal_g_installed_version.stderr is search("command not found") or wal_g_installed_version.stdout != wal_g_version) tags: wal-g, wal_g, wal_g_install From cacbe6e262a5563737b34af9610c201d9ce2b6b4 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 1 Oct 2024 11:24:13 +0300 Subject: [PATCH 09/10] test wal-g install --- automation/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/vars/main.yml b/automation/vars/main.yml index 9646c72ca..9ec6f289f 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -502,7 +502,7 @@ pg_probackup_restore_command: "{{ pg_probackup_command_parts | join('') }}" pg_probackup_patroni_cluster_bootstrap_command: "{{ pg_probackup_command_parts | join('') }}" # WAL-G -wal_g_install: false # or 'true' +wal_g_install: true # or 'true' wal_g_version: "3.0.3" wal_g_installation_method: "binary" # or "src" to build from source code wal_g_path: "/usr/local/bin/wal-g --config {{ postgresql_home_dir }}/.walg.json" From 8863f30243e407321f66e541b6bc1354fbc3b380 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik Date: Tue, 1 Oct 2024 12:17:31 +0300 Subject: [PATCH 10/10] wal_g_install: false (by default) --- automation/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/vars/main.yml b/automation/vars/main.yml index 9ec6f289f..9646c72ca 100644 --- a/automation/vars/main.yml +++ b/automation/vars/main.yml @@ -502,7 +502,7 @@ pg_probackup_restore_command: "{{ pg_probackup_command_parts | join('') }}" pg_probackup_patroni_cluster_bootstrap_command: "{{ pg_probackup_command_parts | join('') }}" # WAL-G -wal_g_install: true # or 'true' +wal_g_install: false # or 'true' wal_g_version: "3.0.3" wal_g_installation_method: "binary" # or "src" to build from source code wal_g_path: "/usr/local/bin/wal-g --config {{ postgresql_home_dir }}/.walg.json"