Merge branch 'master' into restore-timeout

README.md

@@ -206,6 +206,7 @@ docker run -d --name pg-console \
   --publish 8080:8080 \
   --env PG_CONSOLE_API_URL=http://localhost:8080/api/v1 \
   --env PG_CONSOLE_AUTHORIZATION_TOKEN=secret_token \
+  --env PG_CONSOLE_DOCKER_IMAGE=vitabaks/postgresql_cluster:latest \
   --volume console_postgres:/var/lib/postgresql \
   --volume /var/run/docker.sock:/var/run/docker.sock \
   --volume /tmp/ansible:/tmp/ansible \

automation/molecule/default/converge.yml

@@ -16,7 +16,7 @@
         dcs_type: "{{ ['etcd', 'consul'] | random }}" # Set 'dcs_type' to either 'etcd' or 'consul' randomly
         consul_node_role: server # if dcs_type: "consul"
         consul_bootstrap_expect: true # if dcs_type: "consul"
-        postgresql_version: "16" # to test custom WAL dir
+        postgresql_version: 16 # to test custom WAL dir
         pgbouncer_processes: 2 # Test multiple pgbouncer processes (so_reuseport)
         patroni_tags: "datacenter=dc1,key1=value1"
         balancer_tags: "datacenter=dc1"

automation/molecule/pg_upgrade/converge.yml

@@ -16,7 +16,7 @@
         dcs_type: "{{ ['etcd', 'consul'] | random }}" # Set 'dcs_type' to either 'etcd' or 'consul' randomly
         consul_node_role: server # if dcs_type: "consul"
         consul_bootstrap_expect: true # if dcs_type: "consul"
-        postgresql_version: "14" # redefine the version to install for the upgrade test
+        postgresql_version: 14 # redefine the version to install for the upgrade test
         pgbouncer_processes: 4 # Test multiple pgbouncer processes (so_reuseport)
         cacheable: true
       delegate_to: localhost
@@ -43,8 +43,8 @@
 
     - name: Set variables for PostgreSQL upgrade test
       ansible.builtin.set_fact:
-        pg_old_version: "14"
-        pg_new_version: "16"
+        pg_old_version: 14
+        pg_new_version: 16
 
     - name: Add repository GPG key
       ansible.builtin.command: "rpm --import https://repo.almalinux.org/almalinux/RPM-GPG-KEY-AlmaLinux-{{ ansible_distribution_major_version }}"

automation/roles/add-repository/tasks/main.yml

@@ -97,7 +97,7 @@
             - ansible_distribution == "OracleLinux"
             - ansible_distribution_major_version is version('8', '>=')
       vars:
-        pg_devel_package: "postgresql{{ postgresql_version | replace('.', '') }}-devel"
+        pg_devel_package: "postgresql{{ postgresql_version | string | replace('.', '') }}-devel"
       when:
         - pg_devel_package in postgresql_packages
 

automation/roles/cloud-resources/defaults/main.yml

@@ -36,7 +36,7 @@ cloud_load_balancer: true  # Create a Load Balancer in the Cloud.
 
 # Backups (if 'pgbackrest_install' or 'wal_g_install' is 'true')
 aws_s3_bucket_create: true  # if 'cloud_provider=aws'
-aws_s3_bucket_name: "{{ patroni_cluster_name }}}-backup"  # Name of the S3 bucket. Bucket naming rules: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
+aws_s3_bucket_name: "{{ patroni_cluster_name }}-backup"  # Name of the S3 bucket. Bucket naming rules: https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
 aws_s3_bucket_region: "{{ server_location }}"  # The AWS region to use.
 aws_s3_bucket_object_lock_enabled: false  # Whether S3 Object Lock to be enabled.
 aws_s3_bucket_encryption: "AES256"  # Describes the default server-side encryption to apply to new objects in the bucket. Choices: "AES256", "aws:kms"

automation/roles/cloud-resources/tasks/azure.yml

@@ -46,33 +46,46 @@
         PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin"
         PIP_BREAK_SYSTEM_PACKAGES: "1"
 
-    # CLI required for task "Add virtual machine IP addresses to Load Balancer backend pool"
-    - name: Check if Azure CLI is installed
-      ansible.builtin.command: az --version
-      register: az_version_result
-      changed_when: false
-      failed_when: false
-
-    # try to install CLI (if not installed)
-    - name: Install Azure CLI
-      community.general.homebrew:
-        name: azure-cli
-        state: present
-      ignore_errors: true
-      when:
-        - az_version_result.rc != 0
-        - ansible_distribution == "MacOSX"
-
-    - name: Install Azure CLI
-      ansible.builtin.shell: >
-        set -o pipefail;
-        curl -sL https://aka.ms/InstallAzureCli | bash
-      args:
-        executable: /bin/bash
-      ignore_errors: true
-      when:
-        - az_version_result.rc != 0
-        - ansible_distribution != "MacOSX"
+    # Azure CLI
+    # Note: required for task "Add virtual machine IP addresses to Load Balancer backend pool"
+    - block:
+        - name: Check if Azure CLI is installed
+          ansible.builtin.command: az --version
+          register: az_version_result
+          changed_when: false
+          failed_when: false
+
+        # try to install CLI (if not installed)
+        - name: Install Azure CLI
+          community.general.homebrew:
+            name: azure-cli
+            state: present
+          ignore_errors: true
+          when:
+            - az_version_result.rc != 0
+            - ansible_distribution == "MacOSX"
+
+        - name: Install Azure CLI
+          ansible.builtin.shell: >
+            set -o pipefail;
+            curl -sL https://aka.ms/InstallAzureCli | bash
+          args:
+            executable: /bin/bash
+          ignore_errors: true
+          when:
+            - az_version_result.rc != 0
+            - ansible_distribution != "MacOSX"
+
+        # login
+        - name: Login to Azure using Service Principal
+          ansible.builtin.shell: |
+            az login --service-principal \
+            --username "{{ lookup('env', 'AZURE_CLIENT_ID') }}" \
+            --password "{{ lookup('env', 'AZURE_SECRET') }}" \
+            --tenant "{{ lookup('env', 'AZURE_TENANT') }}"
+          args:
+            executable: /bin/bash
+      when: cloud_load_balancer | bool
   delegate_to: 127.0.0.1
   become: false
   run_once: true

automation/roles/cloud-resources/tasks/gcp.yml

@@ -261,6 +261,7 @@
         metadata:
           ssh-keys: "root:{{ ssh_key_content }}"
         scheduling:
+          on_host_maintenance: "{{ 'TERMINATE' if (server_spot | bool or server_type is search('metal')) else 'MIGRATE' }}"
           preemptible: "{{ server_spot | default(gcp_compute_instance_preemptible | default(false)) | bool }}"
         tags:
           items:
@@ -468,7 +469,7 @@
     host: "{{ item.networkInterfaces[0].accessConfigs[0].natIP }}"
     port: 22
     delay: 5
-    timeout: 300
+    timeout: "{{ 1800 if server_type is search('metal') else 300 }}"  # timeout 30 minutes for bare metal instances and 5 minutes for regular VMs
   loop: "{{ server_result.results }}"
   loop_control:
     label: "{{ item.networkInterfaces[0].accessConfigs[0].natIP | default('N/A') }}"

automation/roles/pgbackrest/stanza-create/tasks/main.yml

@@ -2,10 +2,10 @@
 
 # Create a stanza locally (if "pgbackrest_repo_host" is not set)
 - block:
-    - name: Get repo1-path value
+    - name: Get repo1-path and repo1_type value
       ansible.builtin.set_fact:
         repo1_path: "{{ pgbackrest_conf['global'] | selectattr('option', 'equalto', 'repo1-path') | map(attribute='value') | list | first }}"
-      when: pgbackrest_repo_type | lower == 'posix'
+        repo1_type: "{{ pgbackrest_conf['global'] | selectattr('option', 'equalto', 'repo1-type') | map(attribute='value') | list | first }}"
 
     - name: "Make sure the {{ repo1_path }} directory exists"
       ansible.builtin.file:
@@ -14,7 +14,7 @@
         owner: postgres
         group: postgres
         mode: "0750"
-      when: repo1_path | default('') | length > 0
+      when: repo1_type | lower == 'posix'
 
     - name: Create stanza "{{ pgbackrest_stanza }}"
       become: true
@@ -24,6 +24,8 @@
       changed_when:
         - stanza_create_result.rc == 0
         - stanza_create_result.stdout is not search("already exists")
+      when: repo1_type | lower == 'posix' or
+            (repo1_type | lower != 'posix' and inventory_hostname == groups['master'][0])  # run only on master if it's not posix
   when:
     - pgbackrest_repo_host | length < 1
     - "'postgres_cluster' in group_names"

automation/roles/upgrade/tasks/pgbouncer_pause.yml

@@ -39,7 +39,7 @@
       where pid <> pg_backend_pid()
       and state <> 'idle'
       and query_start < clock_timestamp() - interval '{{ pg_slow_active_query_treshold }} ms'
-      {{ "and backend_type = 'client backend'" if pg_old_version is version('10', '>=') else '' }}
+      {{ "and backend_type = 'client backend'" if pg_old_version | string is version('10', '>=') else '' }}
     pg_slow_active_terminate_query: >-
       select
         clock_timestamp(),
@@ -50,7 +50,7 @@
       where pid <> pg_backend_pid()
       and state <> 'idle'
       and query_start < clock_timestamp() - interval '{{ pg_slow_active_query_treshold_to_terminate }} ms'
-      {{ "and backend_type = 'client backend'" if pg_old_version is version('10', '>=') else '' }}
+      {{ "and backend_type = 'client backend'" if pg_old_version | string is version('10', '>=') else '' }}
     pgb_unix_socket_dirs: >-
       {% set unix_socket_dir = ['/var/run/pgbouncer'] %}
       {%- for idx in range(1, pgbouncer_processes | default(1) | int) -%}

automation/roles/upgrade/tasks/post_upgrade.yml

@@ -34,7 +34,7 @@
 # if pg_new_wal_dir is defined
 - name: Delete the old PostgreSQL WAL directory
   ansible.builtin.file:
-    path: "{{ postgresql_wal_dir | regex_replace('(/$)', '') | regex_replace(postgresql_version, pg_old_version) }}"
+    path: "{{ postgresql_wal_dir | regex_replace('(/$)', '') | replace(postgresql_version | string, pg_old_version | string) }}"
     state: absent
   when:
     - pg_current_datadir is success
@@ -48,14 +48,14 @@
   ansible.builtin.package:
     name: "{{ item }}"
     state: absent
-  loop: "{{ postgresql_packages | regex_replace(postgresql_version, pg_old_version) }}"
+  loop: "{{ postgresql_packages | replace(postgresql_version | string, pg_old_version | string) }}"
   register: package_remove
   until: package_remove is success
   delay: 5
   retries: 3
   ignore_errors: true  # show the error and continue the playbook execution
   when:
-    - item is search(pg_old_version)
+    - item | string is search(pg_old_version | string)
     - pg_old_packages_remove | bool
     - ansible_os_family == "RedHat"
 
@@ -67,14 +67,14 @@
     name: "{{ item }}"
     state: absent
     purge: true
-  loop: "{{ postgresql_packages | regex_replace(postgresql_version, pg_old_version) }}"
+  loop: "{{ postgresql_packages | replace(postgresql_version | string, pg_old_version | string) }}"
   register: apt_remove
   until: apt_remove is success
   delay: 5
   retries: 3
   ignore_errors: true  # show the error and continue the playbook execution
   when:
-    - item is search(pg_old_version)
+    - item | string is search(pg_old_version | string)
     - pg_old_packages_remove | bool
     - ansible_os_family == "Debian"
 
@@ -159,14 +159,14 @@
     - name: "WAL-G | Update PostgreSQL data directory path in .walg.json"
       ansible.builtin.replace:
         path: "{{ postgresql_home_dir }}/.walg.json"
-        regexp: "{{ postgresql_data_dir | regex_replace(postgresql_version, pg_old_version) }}"
-        replace: "{{ postgresql_data_dir | regex_replace(postgresql_version, pg_new_version) }}"
+        regexp: "{{ postgresql_data_dir | replace(postgresql_version | string, pg_old_version | string) }}"
+        replace: "{{ postgresql_data_dir | replace(postgresql_version | string, pg_new_version | string) }}"
 
     - name: "WAL-G | Update PostgreSQL data directory path in cron jobs"
       ansible.builtin.replace:
         path: "{{ wal_g_cron_jobs[0].file | default('/etc/cron.d/walg') }}"
-        regexp: "{{ postgresql_data_dir | regex_replace(postgresql_version, pg_old_version) }}"
-        replace: "{{ postgresql_data_dir | regex_replace(postgresql_version, pg_new_version) }}"
+        regexp: "{{ postgresql_data_dir | replace(postgresql_version | string, pg_old_version | string) }}"
+        replace: "{{ postgresql_data_dir | replace(postgresql_version | string, pg_new_version | string) }}"
   become: true
   become_user: root
   ignore_errors: true  # show the error and continue the playbook execution

automation/roles/upgrade/tasks/pre_checks.yml

@@ -7,8 +7,8 @@
     msg:
       - "One or more required variables have empty values."
       - "Please specify a value for the variables: pg_old_version, pg_new_version"
-  failed_when: pg_old_version | length < 1 or pg_new_version | length < 1
-  when: pg_old_version | length < 1 or pg_new_version | length < 1
+  failed_when: pg_old_version | string | length < 1 or pg_new_version | string | length < 1
+  when: pg_old_version | string | length < 1 or pg_new_version | string | length < 1
 
 # Stop, if the directories of the old and new versions are the same
 - name: "Make sure that the old and new data and config directories do not match"
@@ -66,7 +66,7 @@
   changed_when: false
   when:
     - inventory_hostname in groups['primary']
-    - pg_old_version is version('10', '>=')
+    - pg_old_version | string is version('10', '>=')
 
 # for compatibility with Postgres 9.x
 - name: '[Pre-Check] Check the current version of PostgreSQL'
@@ -77,11 +77,11 @@
   changed_when: false
   when:
     - inventory_hostname in groups['primary']
-    - pg_old_version is version('10', '<')
+    - pg_old_version | string is version('10', '<')
 
 - name: "Set variable 'current_pg_version'"
   ansible.builtin.set_fact:
-    current_pg_version: "{{ pg_current_version.stdout if pg_old_version is version('10', '>=') else pg_current_version_9x.stdout }}"
+    current_pg_version: "{{ pg_current_version.stdout if pg_old_version | string is version('10', '>=') else pg_current_version_9x.stdout }}"
   when:
     - inventory_hostname in groups['primary']
 
@@ -168,7 +168,7 @@
   delay: 5
   when:
     - inventory_hostname in groups['primary']
-    - pg_old_version is version('10', '>=')
+    - pg_old_version | string is version('10', '>=')
 
 # Stop, if replication lag is high
 - name: "Pre-Check error. High replication lag"
@@ -195,7 +195,7 @@
   delay: 5
   when:
     - inventory_hostname in groups['primary']
-    - pg_old_version is version('10', '<')
+    - pg_old_version | string is version('10', '<')
 
 # Stop, if replication lag is high (for 9x)
 - name: "Pre-Check error. High replication lag"
@@ -223,7 +223,7 @@
   until: pg_long_transactions.stdout | length < 1
   retries: 30 # 1 minute
   delay: 2
-  when: pg_old_version is version('10', '>=')
+  when: pg_old_version | string is version('10', '>=')
 
 # Stop, if long-running transactions detected
 - block:
@@ -254,7 +254,7 @@
   until: pg_long_transactions_9x.stdout | length < 1
   retries: 30 # 1 minute
   delay: 2
-  when: pg_old_version is version('10', '<')
+  when: pg_old_version | string is version('10', '<')
 
 # Stop, if long-running transactions detected (for 9x)
 - block:

automation/roles/upgrade/tasks/rollback.yml

@@ -83,7 +83,7 @@
       - "The old cluster will need to be restored from backup."
   when:
     - inventory_hostname in groups['primary']
-    - pg_control_version.stdout == pg_new_version | replace('.', '')
+    - pg_control_version.stdout == pg_new_version | string | replace('.', '')
 
 # Restore the old Patroni configuration
 - name: '[Rollback] Restore the old patroni.yml configuration file'

automation/roles/upgrade/tasks/statistics.yml

@@ -39,7 +39,7 @@
       poll: 0
       register: pg_terminator_analyze
       ignore_errors: true  # ignore errors if the task runs for over an 'vacuumdb_analyze_timeout'.
-      when: pg_new_version is version('9.6', '>=')
+      when: pg_new_version | string is version('9.6', '>=')
 
     # Monitor long-running transactions and terminate them (for more than 'vacuumdb_analyze_terminate_treshold')
     - name: "pg_terminator: Monitor and terminate the long-running transactions (more than {{ max_tx_sec }} seconds) during collecting statistics"
@@ -68,7 +68,7 @@
       ignore_errors: true  # ignore errors if the task runs for over an 'vacuumdb_analyze_timeout'.
       vars:
         max_tx_sec: "{{ vacuumdb_analyze_terminate_treshold }}"
-      when: pg_new_version is version('10', '>=') and vacuumdb_analyze_terminate_treshold | int > 0
+      when: pg_new_version | string is version('10', '>=') and vacuumdb_analyze_terminate_treshold | int > 0
 
     # ANALYZE
     - name: "Run vacuumdb to analyze the PostgreSQL databases"

automation/roles/upgrade/tasks/stop_services.yml

@@ -29,7 +29,7 @@
   failed_when: false
   when:
     - inventory_hostname in groups['primary']
-    - pg_old_version is version('10', '>=')
+    - pg_old_version | string is version('10', '>=')
 
 # Stop, if replication lag is high
 - block:
@@ -62,7 +62,7 @@
   failed_when: false
   when:
     - inventory_hostname in groups['primary']
-    - pg_old_version is version('10', '<')
+    - pg_old_version | string is version('10', '<')
 
 # Stop, if replication lag is high (for 9x)
 - block:

automation/roles/upgrade/tasks/upgrade_secondary.yml

@@ -49,8 +49,8 @@
   become_user: postgres
   when:
     - inventory_hostname in groups['primary']
-    - pg_old_datadir|dirname == pg_upper_datadir + '/' + pg_old_version
-    - pg_new_datadir|dirname == pg_upper_datadir + '/' + pg_new_version
+    - pg_old_datadir|dirname == pg_upper_datadir + '/' + (pg_old_version | string)
+    - pg_new_datadir|dirname == pg_upper_datadir + '/' + (pg_new_version | string)
 
 # If the source and target directories are non-versioned directories
 # (example: /pgdata/main -> /pgdata/main<pg_new_version>)
@@ -81,8 +81,8 @@
   become_user: postgres
   when:
     - inventory_hostname in groups['primary']
-    - pg_old_datadir|dirname != pg_upper_datadir + '/' + pg_old_version
-    - pg_new_datadir|dirname != pg_upper_datadir + '/' + pg_new_version
+    - pg_old_datadir|dirname != pg_upper_datadir + '/' + (pg_old_version | string)
+    - pg_new_datadir|dirname != pg_upper_datadir + '/' + (pg_new_version | string)
 
 # Tablespaces (if exists)
 - block: