From d9f4914e0d78fa45c7a76526a660faf94213b3df Mon Sep 17 00:00:00 2001
From: transcaffeine <transcaffeine@finallycoffee.eu>
Date: Sun, 29 Nov 2020 15:20:22 +0100
Subject: [PATCH 01/79] WIP: postgres: create databases for all services

If a service is enabled, a database for it is created in postgres with a uniqque password. The service can then use this database for data storage instead of relying on sqlite.
---
 group_vars/matrix_servers                     | 64 +++++++++++++++++++
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  3 +
 .../templates/config.yaml.j2                  |  2 +-
 .../templates/config.yaml.j2                  |  2 +-
 .../templates/config.yaml.j2                  |  2 +-
 .../templates/config.yaml.j2                  |  2 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../templates/config.yaml.j2                  |  4 +-
 .../matrix-dimension/templates/config.yaml.j2 |  2 +-
 .../matrix-postgres/tasks/setup_postgres.yml  | 30 +++++++++
 16 files changed, 118 insertions(+), 21 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 78a038517ed..8a099275d7b 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -889,6 +889,70 @@ matrix_postgres_connection_username: "synapse"
 matrix_postgres_connection_password: "synapse-password"
 matrix_postgres_db_name: "homeserver"
 
+matrix_postgres_additional_databases: |
+  {{
+    ([{
+      name: 'matrix_appservice_discord',
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
+    }] if matrix_appservice_discord_enabled else [])
+    + ([{
+      name: 'matrix_appservice_slack'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
+    }] if matrix_appservice_slack_enabled else [])
+    + ([{
+      name: 'matrix_appservice_irc'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
+    }] if matrix_appservice_irc_enabled else [])
+    + ([{
+      name: 'mautrix-bridge-facebook'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
+    }] if matrix_mautrix_facebook_enabled else [])
+    + ([{
+      name: 'mautrix_bridge_hangouts'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
+    }] if matrix_mautrix_hangouts_enabled else [])
+    + ([{
+      name: 'mautrix_bridge_telegram'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
+    }] if matrix_mautrix_telegram_enabled else [])
+    + ([{
+      name: 'mautrix_bridge_whatsapp'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
+    }] if matrix_mautrix_whatsapp_enabled else [])
+    + ([{
+      name: 'matrix_bridge_sms'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
+    }] if matrix_sms_bridge_enabled else [])
+    + ([{
+      name: 'matrix_puppet_skype'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
+    }] if matrix_mx_puppet_skype_enabled else [])
+    + ([{
+      name: 'matrix_puppet_slack'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
+    }] if matrix_mx_puppet_slack_enabled else [])
+    + ([{
+      name: 'matrix_puppet_twitter'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
+    }] if matrix_mx_puppet_twitter_enabled else [])
+    + ([{
+      name: 'matrix_puppet_instagram'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
+    ] if matrix_mx_puppet_instagram_enabled else [])
+    + ([{
+      name: 'matrix_puppet_discord'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
+    }] if matrix_mx_puppet_discord_enabled else [])
+    + ([{
+      name: 'matrix_puppet_steam'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
+    }] if matrix_mx_puppet_steam_enabled else [])
+    + ([{
+      name: 'matrix_dimension'
+      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
+    }] if matrix_dimension_enabled else [])
+  }}
+
 ######################################################################
 #
 # /matrix-postgres
diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
index b99f522abb9..de4182d9a4a 100644
--- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
@@ -58,8 +58,8 @@ database:
   # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite
   # WARNING: You will almost certainly be fine with sqlite unless your bridge
   # is in heavy demand and you suffer from IO slowness.
-  filename: "/data/discord.db"
-  # connString: "postgresql://user:password@localhost/database_name"
+  #filename: "/data/discord.db"
+  connString: "postgresql://matrix_appservice_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_appservice_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_discord"
 room:
   # Set the default visibility of alias rooms, defaults to "public".
   # One of: "public", "private"
diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
index 3daa1898764..83b71835872 100644
--- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
@@ -127,8 +127,8 @@ advanced:
 # Use an external database to store bridge state.
 database:
   # database engine (must be 'postgres' or 'nedb'). Default: nedb
-  engine: "nedb"
+  engine: "postgres"
   # Either a PostgreSQL connection string, or a path to the NeDB storage directory.
   # For postgres, it must start with postgres://
   # For NeDB, it must start with nedb://. The path is relative to the project directory.
-  connectionString: "nedb:///data"
+  connectionString: "postgres://matrix_appservice_irc:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_irc') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_irc"
diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
index 8f48d3178ca..6c491134af5 100644
--- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
@@ -10,5 +10,8 @@ homeserver:
     server_name: "{{ matrix_domain }}"
 
 dbdir: "/data"
+db:
+    engine: "postgres"
+    connectionString: "postgresql://matrix_appservice_slack:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_slack"
 
 matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"
diff --git a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
index 9d8de2d5ba5..304571c136e 100644
--- a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-facebook.db
+    database: sqlite://matrix_bridge_facebook:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_facebook') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_facebook
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     public:
diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
index f274b2034c6..e4e59ad26d9 100644
--- a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-hangouts.db
+    database: postgres://mautrix_bridge_hangouts:{{ matrix_additional_databases | selectattr('name', 'equalto', 'mautrix_bridge_hangouts') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_hangouts
 
     # The unique ID of this appservice.
     id: hangouts
diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
index 1a4ac43e9b5..65f17d338a2 100644
--- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite:////data/mautrix-telegram.db
+    database: postgres://mautrix_bridge_telegram:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_bridge_telegram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_telegram
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in
diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
index a527a1884fc..93956049ced 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
@@ -23,7 +23,7 @@ appservice:
         # The database URI.
         #   SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string
         #   Postgres: Connection string. For example, postgres://user:password@host/database
-        uri: mautrix-whatsapp.db
+        uri: postgres://matrix_bridge_whatsapp@{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_whatsapp') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_whatsapp
         # Maximum number of connections. Mostly relevant for Postgres.
         max_open_conns: 20
         max_idle_conns: 2
diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
index 2c70379617d..88c591e2f63 100644
--- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
@@ -105,10 +105,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_discord?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
index 634fbaecdc9..2d1891957db 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
@@ -49,10 +49,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_instagram:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_instagram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_instagram?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
index c7b5c870e62..a54ca758b34 100644
--- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
@@ -73,10 +73,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_skype:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_skype') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_skype?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 provisioning:
   # Regex of Matrix IDs allowed to use the puppet bridge
diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
index b6e88784831..7f1b21d785b 100644
--- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
@@ -63,10 +63,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_slack:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_slack?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
index d08982ca9f7..14f86319612 100644
--- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
@@ -66,10 +66,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_steam:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_steam') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_steam?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
index 7d3033b39d9..853d23fc26c 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
@@ -59,10 +59,10 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  #connString: "postgres://user:pass@localhost/dbname?sslmode=disable"
+  connString: "postgres://matrix_puppet_twitter:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_twitter') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_twitter?sslmode=disable"
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  #filename: /data/database.db
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-dimension/templates/config.yaml.j2 b/roles/matrix-dimension/templates/config.yaml.j2
index a05b6c35a63..b7ca1ee3b8d 100644
--- a/roles/matrix-dimension/templates/config.yaml.j2
+++ b/roles/matrix-dimension/templates/config.yaml.j2
@@ -44,7 +44,7 @@ widgetBlacklist:
 
 # Where the database for Dimension is
 database:
-  file: "dimension.db"
+  uri: "postgres://matrix_dimension:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_dimension') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_dimension"
 
 # Display settings that apply to self-hosted go-neb instances
 goneb:
diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml
index f186bdca4c0..85a8604c69c 100644
--- a/roles/matrix-postgres/tasks/setup_postgres.yml
+++ b/roles/matrix-postgres/tasks/setup_postgres.yml
@@ -162,3 +162,33 @@
     - matrix-change-user-admin-status
     - matrix-postgres-update-user-password-hash
   when: "not matrix_postgres_enabled|bool"
+
+# Create additional databases
+- name: Retrieve IP of postgres container
+  shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
+  register: matirx_postgres_container_ip
+
+- name: Create additional users in postgres
+  postgresql_user:
+    name: "{{ item.name }}"
+    password: "{{ item.pass }}"
+    login_host: "{{ matrx_postgres_container_ip.stdout }}"
+    login_port: 5432
+    login_user: "{{ matrix_postgres_connection_username }}"
+    login_password: "{{ matrix_postgres_connection_password }}"
+    login_db: "{{ matrix_postgres_db_name }}"
+  loop: matrix_postgres_additional_databases
+  when: matrix_postgres_enabed|bool
+
+- name: Create additional users in postgres
+  postgresql_db:
+    name: "{{ item.name }}"
+    owner: "{{ item.name }}"
+    lc_ctype: 'C'
+    lc_collate: 'C'
+    login_host: "{{ matrx_postgres_container_ip.stdout }}"
+    login_port: 5432
+    login_user: "{{ matrix_postgres_connection_username }}"
+    login_password: "{{ matrix_postgres_connection_password }}"
+  loop: matrix_postgres_additional_databases
+  when: matrix_postgres_enabled|bool

From dac0d3a682d6a288dea3c22115c3d8385e79c720 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 21:07:16 +0200
Subject: [PATCH 02/79] Add default matrix_postgres_additional_databases

---
 roles/matrix-postgres/defaults/main.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index ec5cb3dca4f..548cac15bbb 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -30,3 +30,13 @@ matrix_postgres_container_extra_arguments: []
 #
 # Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:5432"), or empty string to not expose.
 matrix_postgres_container_postgres_bind_port: ""
+
+# A list of additional (databases and their credentials) to create.
+#
+# Example:
+# matrix_postgres_additional_databases:
+#   - name: matrix_appservice_discord
+#     pass: some-password
+#   - name: matrix_appservice_slack
+#     pass: some-password
+matrix_postgres_additional_databases: []

From 527d5f57d58ad7c1feba743fc99fe8508cc90534 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 21:40:32 +0200
Subject: [PATCH 03/79] Relocate Postgres additional database creation logic

Moving it above the "uninstalling" set of tasks is better.
Extracting it out to another file at the same time, for readability,
especially given that it will probably have to become more complex in
the future (potentially installing `jq`, etc.)
---
 .../matrix-postgres/tasks/setup_postgres.yml  | 37 ++++---------------
 .../util/create_additional_databases.yml      | 28 ++++++++++++++
 2 files changed, 35 insertions(+), 30 deletions(-)
 create mode 100644 roles/matrix-postgres/tasks/util/create_additional_databases.yml

diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml
index 85a8604c69c..6a0f9854805 100644
--- a/roles/matrix-postgres/tasks/setup_postgres.yml
+++ b/roles/matrix-postgres/tasks/setup_postgres.yml
@@ -113,6 +113,13 @@
     daemon_reload: yes
   when: "matrix_postgres_enabled|bool and matrix_postgres_systemd_service_result.changed"
 
+- include_tasks:
+    file: "{{ role_path }}/tasks/util/create_additional_databases.yml"
+    apply:
+      tags:
+        - always
+  when: "matrix_postgres_enabled|bool"
+
 #
 # Tasks related to getting rid of the internal postgres server (if it was previously enabled)
 #
@@ -162,33 +169,3 @@
     - matrix-change-user-admin-status
     - matrix-postgres-update-user-password-hash
   when: "not matrix_postgres_enabled|bool"
-
-# Create additional databases
-- name: Retrieve IP of postgres container
-  shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
-  register: matirx_postgres_container_ip
-
-- name: Create additional users in postgres
-  postgresql_user:
-    name: "{{ item.name }}"
-    password: "{{ item.pass }}"
-    login_host: "{{ matrx_postgres_container_ip.stdout }}"
-    login_port: 5432
-    login_user: "{{ matrix_postgres_connection_username }}"
-    login_password: "{{ matrix_postgres_connection_password }}"
-    login_db: "{{ matrix_postgres_db_name }}"
-  loop: matrix_postgres_additional_databases
-  when: matrix_postgres_enabed|bool
-
-- name: Create additional users in postgres
-  postgresql_db:
-    name: "{{ item.name }}"
-    owner: "{{ item.name }}"
-    lc_ctype: 'C'
-    lc_collate: 'C'
-    login_host: "{{ matrx_postgres_container_ip.stdout }}"
-    login_port: 5432
-    login_user: "{{ matrix_postgres_connection_username }}"
-    login_password: "{{ matrix_postgres_connection_password }}"
-  loop: matrix_postgres_additional_databases
-  when: matrix_postgres_enabled|bool
diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
new file mode 100644
index 00000000000..ec37ea6311f
--- /dev/null
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -0,0 +1,28 @@
+---
+
+- name: Retrieve IP of postgres container
+  shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
+  register: matrix_postgres_container_ip
+
+- name: Create additional users in postgres
+  postgresql_user:
+    name: "{{ item.name }}"
+    password: "{{ item.pass }}"
+    login_host: "{{ matrix_postgres_container_ip.stdout }}"
+    login_port: 5432
+    login_user: "{{ matrix_postgres_connection_username }}"
+    login_password: "{{ matrix_postgres_connection_password }}"
+    login_db: "{{ matrix_postgres_db_name }}"
+  loop: matrix_postgres_additional_databases
+
+- name: Create additional users in postgres
+  postgresql_db:
+    name: "{{ item.name }}"
+    owner: "{{ item.name }}"
+    lc_ctype: 'C'
+    lc_collate: 'C'
+    login_host: "{{ matrix_postgres_container_ip.stdout }}"
+    login_port: 5432
+    login_user: "{{ matrix_postgres_connection_username }}"
+    login_password: "{{ matrix_postgres_connection_password }}"
+  loop: matrix_postgres_additional_databases

From 0641106370aed272e16cecaf2cf17754d30eb6c6 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 21:43:53 +0200
Subject: [PATCH 04/79] Allow username of additional Postgres databases to be
 different

We'll most likely use one that matches the database name, but
it's better to have it configurable.
---
 group_vars/matrix_servers                       | 17 ++++++++++++++++-
 roles/matrix-postgres/defaults/main.yml         |  6 ++++--
 .../tasks/util/create_additional_databases.yml  |  4 ++--
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 8a099275d7b..f131ab9e1ce 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -893,62 +893,77 @@ matrix_postgres_additional_databases: |
   {{
     ([{
       name: 'matrix_appservice_discord',
+      username: 'matrix_appservice_discord',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
     }] if matrix_appservice_discord_enabled else [])
     + ([{
       name: 'matrix_appservice_slack'
+      username: 'matrix_appservice_slack',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
     }] if matrix_appservice_slack_enabled else [])
     + ([{
       name: 'matrix_appservice_irc'
+      username: 'matrix_appservice_irc',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
     }] if matrix_appservice_irc_enabled else [])
     + ([{
-      name: 'mautrix-bridge-facebook'
+      name: 'mautrix_bridge_facebook'
+      username: 'mautrix_bridge_facebook',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
     }] if matrix_mautrix_facebook_enabled else [])
     + ([{
       name: 'mautrix_bridge_hangouts'
+      username: 'mautrix_bridge_hangouts',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
     }] if matrix_mautrix_hangouts_enabled else [])
     + ([{
       name: 'mautrix_bridge_telegram'
+      username: 'mautrix_bridge_telegram',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
     }] if matrix_mautrix_telegram_enabled else [])
     + ([{
       name: 'mautrix_bridge_whatsapp'
+      username: 'mautrix_bridge_whatsapp',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
     }] if matrix_mautrix_whatsapp_enabled else [])
     + ([{
       name: 'matrix_bridge_sms'
+      username: 'matrix_bridge_sms',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
     }] if matrix_sms_bridge_enabled else [])
     + ([{
       name: 'matrix_puppet_skype'
+      username: 'matrix_puppet_skype',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
     }] if matrix_mx_puppet_skype_enabled else [])
     + ([{
       name: 'matrix_puppet_slack'
+      username: 'matrix_puppet_slack',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
     }] if matrix_mx_puppet_slack_enabled else [])
     + ([{
       name: 'matrix_puppet_twitter'
+      username: 'matrix_puppet_twitter',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
     }] if matrix_mx_puppet_twitter_enabled else [])
     + ([{
       name: 'matrix_puppet_instagram'
+      username: 'matrix_puppet_instagram',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
     ] if matrix_mx_puppet_instagram_enabled else [])
     + ([{
       name: 'matrix_puppet_discord'
+      username: 'matrix_puppet_discord',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
     }] if matrix_mx_puppet_discord_enabled else [])
     + ([{
       name: 'matrix_puppet_steam'
+      username: 'matrix_puppet_steam',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
     }] if matrix_mx_puppet_steam_enabled else [])
     + ([{
       name: 'matrix_dimension'
+      username: 'matrix_dimension',
       pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
     }] if matrix_dimension_enabled else [])
   }}
diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 548cac15bbb..2645d4b0a34 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -36,7 +36,9 @@ matrix_postgres_container_postgres_bind_port: ""
 # Example:
 # matrix_postgres_additional_databases:
 #   - name: matrix_appservice_discord
-#     pass: some-password
+#     username: matrix_appservice_discord
+#     pass: some_password
 #   - name: matrix_appservice_slack
-#     pass: some-password
+#     username: matrix_appservice_slack
+#     pass: some_password
 matrix_postgres_additional_databases: []
diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index ec37ea6311f..0532c99d429 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -6,7 +6,7 @@
 
 - name: Create additional users in postgres
   postgresql_user:
-    name: "{{ item.name }}"
+    name: "{{ item.username }}"
     password: "{{ item.pass }}"
     login_host: "{{ matrix_postgres_container_ip.stdout }}"
     login_port: 5432
@@ -18,7 +18,7 @@
 - name: Create additional users in postgres
   postgresql_db:
     name: "{{ item.name }}"
-    owner: "{{ item.name }}"
+    owner: "{{ item.username }}"
     lc_ctype: 'C'
     lc_collate: 'C'
     login_host: "{{ matrix_postgres_container_ip.stdout }}"

From d251764c16303dea542326746a0e13a46d13fda2 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 22:19:54 +0200
Subject: [PATCH 05/79] Fix syntax issues in
 matrix_postgres_additional_databases

Quotes are necessary around dictionary field names.

There was a missing `}` as well.
---
 group_vars/matrix_servers | 92 +++++++++++++++++++--------------------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index f131ab9e1ce..11c87a6491d 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -892,79 +892,79 @@ matrix_postgres_db_name: "homeserver"
 matrix_postgres_additional_databases: |
   {{
     ([{
-      name: 'matrix_appservice_discord',
-      username: 'matrix_appservice_discord',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
+      'name': 'matrix_appservice_discord',
+      'username': 'matrix_appservice_discord',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
     }] if matrix_appservice_discord_enabled else [])
     + ([{
-      name: 'matrix_appservice_slack'
-      username: 'matrix_appservice_slack',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
+      'name': 'matrix_appservice_slack',
+      'username': 'matrix_appservice_slack',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
     }] if matrix_appservice_slack_enabled else [])
     + ([{
-      name: 'matrix_appservice_irc'
-      username: 'matrix_appservice_irc',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
+      'name': 'matrix_appservice_irc',
+      'username': 'matrix_appservice_irc',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
     }] if matrix_appservice_irc_enabled else [])
     + ([{
-      name: 'mautrix_bridge_facebook'
-      username: 'mautrix_bridge_facebook',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
+      'name': 'mautrix_bridge_facebook',
+      'username': 'mautrix_bridge_facebook',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
     }] if matrix_mautrix_facebook_enabled else [])
     + ([{
-      name: 'mautrix_bridge_hangouts'
-      username: 'mautrix_bridge_hangouts',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
+      'name': 'mautrix_bridge_hangouts',
+      'username': 'mautrix_bridge_hangouts',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
     }] if matrix_mautrix_hangouts_enabled else [])
     + ([{
-      name: 'mautrix_bridge_telegram'
-      username: 'mautrix_bridge_telegram',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
+      'name': 'mautrix_bridge_telegram',
+      'username': 'mautrix_bridge_telegram',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
     }] if matrix_mautrix_telegram_enabled else [])
     + ([{
-      name: 'mautrix_bridge_whatsapp'
-      username: 'mautrix_bridge_whatsapp',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
+      'name': 'mautrix_bridge_whatsapp',
+      'username': 'mautrix_bridge_whatsapp',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
     }] if matrix_mautrix_whatsapp_enabled else [])
     + ([{
-      name: 'matrix_bridge_sms'
-      username: 'matrix_bridge_sms',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
+      'name': 'matrix_bridge_sms',
+      'username': 'matrix_bridge_sms',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      name: 'matrix_puppet_skype'
-      username: 'matrix_puppet_skype',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
+      'name': 'matrix_puppet_skype',
+      'username': 'matrix_puppet_skype',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
     }] if matrix_mx_puppet_skype_enabled else [])
     + ([{
-      name: 'matrix_puppet_slack'
-      username: 'matrix_puppet_slack',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
+      'name': 'matrix_puppet_slack',
+      'username': 'matrix_puppet_slack',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
     }] if matrix_mx_puppet_slack_enabled else [])
     + ([{
-      name: 'matrix_puppet_twitter'
-      username: 'matrix_puppet_twitter',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
+      'name': 'matrix_puppet_twitter',
+      'username': 'matrix_puppet_twitter',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
     }] if matrix_mx_puppet_twitter_enabled else [])
     + ([{
-      name: 'matrix_puppet_instagram'
-      username: 'matrix_puppet_instagram',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
-    ] if matrix_mx_puppet_instagram_enabled else [])
+      'name': 'matrix_puppet_instagram',
+      'username': 'matrix_puppet_instagram',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
+    }] if matrix_mx_puppet_instagram_enabled else [])
     + ([{
-      name: 'matrix_puppet_discord'
-      username: 'matrix_puppet_discord',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
+      'name': 'matrix_puppet_discord',
+      'username': 'matrix_puppet_discord',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
     }] if matrix_mx_puppet_discord_enabled else [])
     + ([{
-      name: 'matrix_puppet_steam'
-      username: 'matrix_puppet_steam',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
+      'name': 'matrix_puppet_steam',
+      'username': 'matrix_puppet_steam',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
     }] if matrix_mx_puppet_steam_enabled else [])
     + ([{
-      name: 'matrix_dimension'
-      username: 'matrix_dimension',
-      pass: matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
+      'name': 'matrix_dimension',
+      'username': 'matrix_dimension',
+      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
     }] if matrix_dimension_enabled else [])
   }}
 

From f47e8a97e6629da02a45d9d75bde3a82ef057bd6 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 22:38:35 +0200
Subject: [PATCH 06/79] Make use of matrix_host_command_docker instead of
 hardcoding

---
 .../matrix-postgres/tasks/util/create_additional_databases.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index 0532c99d429..424be6264fe 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -1,7 +1,6 @@
 ---
-
 - name: Retrieve IP of postgres container
-  shell: "docker inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
+  shell: "{{ matrix_host_command_docker }} inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
   register: matrix_postgres_container_ip
 
 - name: Create additional users in postgres

From e2952f16f7a97b2883ea4c00c6bc4451c71c785d Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 22:45:48 +0200
Subject: [PATCH 07/79] Determine matrix-postgres IP address without relying on
 jq

To avoid needing to have `jq` installed on the machine, we could:
- try to run jq in a Docker container using some small image providing
that
- better yet, avoid `jq` altogether
---
 .../matrix-postgres/tasks/util/create_additional_databases.yml  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index 424be6264fe..0864d8f824e 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -1,6 +1,6 @@
 ---
 - name: Retrieve IP of postgres container
-  shell: "{{ matrix_host_command_docker }} inspect matrix-postgres | jq -r '.[0].NetworkSettings.Networks.{{ matrix_docker_network }}.IPAddress'"
+  command: "{{ matrix_host_command_docker }} inspect matrix-postgres --format='{% raw %}{{ .NetworkSettings.Networks.{% endraw %}{{ matrix_docker_network }}{% raw %}.IPAddress }}{% endraw %}'"
   register: matrix_postgres_container_ip
 
 - name: Create additional users in postgres

From c765ceb270b7377e799a32db383eef39257d43e8 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 22:56:56 +0200
Subject: [PATCH 08/79] Prevent weird loop error

> Invalid data passed to 'loop', it requires a list, got this instead: matrix_postgres_additional_databases. Hint: If you passed a list/dict of just one element, try adding wantlist=True to your lookup invocation or use q/query instead of lookup.

Well, or working around it, as I've done in this commit (which seems
more sane than `wantlist=True` stuff).
---
 .../tasks/util/create_additional_databases.yml               | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index 0864d8f824e..0b40cabfa8e 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -1,4 +1,5 @@
 ---
+
 - name: Retrieve IP of postgres container
   command: "{{ matrix_host_command_docker }} inspect matrix-postgres --format='{% raw %}{{ .NetworkSettings.Networks.{% endraw %}{{ matrix_docker_network }}{% raw %}.IPAddress }}{% endraw %}'"
   register: matrix_postgres_container_ip
@@ -12,7 +13,7 @@
     login_user: "{{ matrix_postgres_connection_username }}"
     login_password: "{{ matrix_postgres_connection_password }}"
     login_db: "{{ matrix_postgres_db_name }}"
-  loop: matrix_postgres_additional_databases
+  loop: "{{ matrix_postgres_additional_databases }}"
 
 - name: Create additional users in postgres
   postgresql_db:
@@ -24,4 +25,4 @@
     login_port: 5432
     login_user: "{{ matrix_postgres_connection_username }}"
     login_password: "{{ matrix_postgres_connection_password }}"
-  loop: matrix_postgres_additional_databases
+  loop: "{{ matrix_postgres_additional_databases }}"

From bbc09d013b8037e784004363b7bdfdcc6e8d13c7 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sun, 13 Dec 2020 23:46:02 +0200
Subject: [PATCH 09/79] Do not execute additional databases creation code if
 not necessary

The tasks in `create_additional_databases.yml` will likely
ensure `matrix-postgres.service` is started, etc.

If no additional databases are defined, we'd rather not execute that
file and all these tasks that it may do in the future.
---
 roles/matrix-postgres/tasks/setup_postgres.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml
index 6a0f9854805..518d1a5f028 100644
--- a/roles/matrix-postgres/tasks/setup_postgres.yml
+++ b/roles/matrix-postgres/tasks/setup_postgres.yml
@@ -118,7 +118,7 @@
     apply:
       tags:
         - always
-  when: "matrix_postgres_enabled|bool"
+  when: "matrix_postgres_enabled|bool and matrix_postgres_additional_databases|length > 0"
 
 #
 # Tasks related to getting rid of the internal postgres server (if it was previously enabled)

From da4cb2f63998c2c994776b3520303e90b735157c Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 00:25:13 +0200
Subject: [PATCH 10/79] Do not use the postgresql_user/postgresql_db modules

While these modules are really nice and helpful, we can't use them
for at least 2 reasons:

- for us, Postgres runs in a container on a private Docker network
(`--network=matrix`) without usually being exposed to the host.
These modules execute on the host so they won't be able to reach it.

- these modules require `psycopg2`, so we need to install it before
using it. This might or might not be its own can of worms.
---
 .../tasks/util/create_additional_database.yml | 34 ++++++++++++++++++
 .../util/create_additional_databases.yml      | 35 ++++++-------------
 .../init-additional-db-user-and-role.sql.j2   | 19 ++++++++++
 3 files changed, 63 insertions(+), 25 deletions(-)
 create mode 100644 roles/matrix-postgres/tasks/util/create_additional_database.yml
 create mode 100644 roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2

diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml
new file mode 100644
index 00000000000..2da505eba42
--- /dev/null
+++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml
@@ -0,0 +1,34 @@
+---
+
+# TODO - ensure `additional_db` contains all keys that we expect
+
+# The SQL statements that we'll run against Postgres are stored in a file that others can't read.
+# This file will be mounted into the container and fed to Postgres.
+# This way, we avoid passing sensitive data around in CLI commands that other users on the system can see.
+- name: Create additional database initialization SQL file for {{ additional_db.name }}
+  template:
+    src: "{{ role_path }}/templates/init-additional-db-user-and-role.sql.j2"
+    dest: "/tmp/matrix-postgres-init-additional-db-user-and-role.sql"
+    mode: 0600
+    owner: "{{ matrix_user_uid }}"
+    group: "{{ matrix_user_gid }}"
+
+- name: Execute Postgres additional database initialization SQL file for {{ additional_db.name }}
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --env-file={{ matrix_postgres_base_path }}/env-postgres-psql
+      --network {{ matrix_docker_network }}
+      --mount type=bind,src=/tmp/matrix-postgres-init-additional-db-user-and-role.sql,dst=/matrix-postgres-init-additional-db-user-and-role.sql,ro
+      --entrypoint=/bin/sh
+      {{ matrix_postgres_docker_image_to_use }}
+      -c
+      'psql -h {{ matrix_postgres_connection_hostname }} --file=/matrix-postgres-init-additional-db-user-and-role.sql'
+
+- name: Delete additional database initialization SQL file for {{ additional_db.name }}
+  file:
+    path: /tmp/matrix-postgres-init-additional-db-user-and-role.sql
+    state: absent
diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index 0b40cabfa8e..51deb228163 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -1,28 +1,13 @@
 ---
 
-- name: Retrieve IP of postgres container
-  command: "{{ matrix_host_command_docker }} inspect matrix-postgres --format='{% raw %}{{ .NetworkSettings.Networks.{% endraw %}{{ matrix_docker_network }}{% raw %}.IPAddress }}{% endraw %}'"
-  register: matrix_postgres_container_ip
+# TODO - we should ensure matrix-postgres.service is started.
+# .. and that if we had just started it, we've given it ample time to initialize,
+# before we attempt to run queries against it.
 
-- name: Create additional users in postgres
-  postgresql_user:
-    name: "{{ item.username }}"
-    password: "{{ item.pass }}"
-    login_host: "{{ matrix_postgres_container_ip.stdout }}"
-    login_port: 5432
-    login_user: "{{ matrix_postgres_connection_username }}"
-    login_password: "{{ matrix_postgres_connection_password }}"
-    login_db: "{{ matrix_postgres_db_name }}"
-  loop: "{{ matrix_postgres_additional_databases }}"
-
-- name: Create additional users in postgres
-  postgresql_db:
-    name: "{{ item.name }}"
-    owner: "{{ item.username }}"
-    lc_ctype: 'C'
-    lc_collate: 'C'
-    login_host: "{{ matrix_postgres_container_ip.stdout }}"
-    login_port: 5432
-    login_user: "{{ matrix_postgres_connection_username }}"
-    login_password: "{{ matrix_postgres_connection_password }}"
-  loop: "{{ matrix_postgres_additional_databases }}"
+- name: Create additional Postgers user and database
+  include_tasks: "{{ role_path }}/tasks/util/create_additional_database.yml"
+  with_items: "{{ matrix_postgres_additional_databases }}"
+  loop_control:
+    loop_var: additional_db
+  # Suppress logging to avoid dumping the credentials to the shell
+  no_log: true
diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
new file mode 100644
index 00000000000..732b01870cd
--- /dev/null
+++ b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
@@ -0,0 +1,19 @@
+-- `CREATE USER` does not support `IF NOT EXISTS`, so we use this workaround to prevent an error and raise a notice instead.
+-- Seen here: https://stackoverflow.com/a/49858797
+DO $$
+BEGIN
+  CREATE USER {{ additional_db.username }};
+  EXCEPTION WHEN DUPLICATE_OBJECT THEN
+  RAISE NOTICE 'not creating role {{ additional_db.username }}, since it already exists';
+END
+$$;
+
+-- This is useful for initial user creation (since we don't assign a password above) and for handling subsequent password changes
+-- TODO - we should escape quotes in the password.
+ALTER ROLE {{ additional_db.username }} PASSWORD '{{ additional_db.pass }}';
+
+-- This will generate an error on subsequent execution
+CREATE DATABASE {{ additional_db.name }} WITH LC_CTYPE 'C' LC_COLLATE 'C' OWNER {{ additional_db.username }};
+
+-- This is useful for changing the database owner subsequently
+ALTER DATABASE {{ additional_db.name }} OWNER TO {{ additional_db.username }};

From 3a037a59935002729dfe7da742721111956eaf0b Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 00:39:38 +0200
Subject: [PATCH 11/79] Ensure additional databases contain all the keys that
 we expect

---
 .../tasks/util/create_additional_database.yml             | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml
index 2da505eba42..a994cc260a4 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_database.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml
@@ -1,6 +1,12 @@
 ---
 
-# TODO - ensure `additional_db` contains all keys that we expect
+# It'd be better if this is belonged to `validate_config.yml`, but it would have to be some loop-within-a-loop there,
+# and that's ugly. We also don't expect this to catch errors often. It's more of a defensive last-minute check.
+- name: Fail if additional database data appears invalid
+  fail:
+    msg: "Additional database definition ({{ additional_db }} lacks a required key: {{ item }}"
+  when: "item not in additional_db"
+  with_items: "{{ ['name', 'username', 'pass'] }}"
 
 # The SQL statements that we'll run against Postgres are stored in a file that others can't read.
 # This file will be mounted into the container and fed to Postgres.

From 46a4034d3ec0b8ae331119e249fc584f7be2b6ed Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 00:43:03 +0200
Subject: [PATCH 12/79] Use "password" for additional Postgres databases, not
 "pass"

Being more explicit sounds better.
---
 group_vars/matrix_servers                     | 30 +++++++++----------
 roles/matrix-postgres/defaults/main.yml       |  4 +--
 .../tasks/util/create_additional_database.yml |  2 +-
 .../init-additional-db-user-and-role.sql.j2   |  2 +-
 4 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 11c87a6491d..3169e18e46a 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -894,77 +894,77 @@ matrix_postgres_additional_databases: |
     ([{
       'name': 'matrix_appservice_discord',
       'username': 'matrix_appservice_discord',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string,
     }] if matrix_appservice_discord_enabled else [])
     + ([{
       'name': 'matrix_appservice_slack',
       'username': 'matrix_appservice_slack',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string,
     }] if matrix_appservice_slack_enabled else [])
     + ([{
       'name': 'matrix_appservice_irc',
       'username': 'matrix_appservice_irc',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string,
     }] if matrix_appservice_irc_enabled else [])
     + ([{
       'name': 'mautrix_bridge_facebook',
       'username': 'mautrix_bridge_facebook',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string,
     }] if matrix_mautrix_facebook_enabled else [])
     + ([{
       'name': 'mautrix_bridge_hangouts',
       'username': 'mautrix_bridge_hangouts',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string,
     }] if matrix_mautrix_hangouts_enabled else [])
     + ([{
       'name': 'mautrix_bridge_telegram',
       'username': 'mautrix_bridge_telegram',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string,
     }] if matrix_mautrix_telegram_enabled else [])
     + ([{
       'name': 'mautrix_bridge_whatsapp',
       'username': 'mautrix_bridge_whatsapp',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string,
     }] if matrix_mautrix_whatsapp_enabled else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
       'name': 'matrix_puppet_skype',
       'username': 'matrix_puppet_skype',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string,
     }] if matrix_mx_puppet_skype_enabled else [])
     + ([{
       'name': 'matrix_puppet_slack',
       'username': 'matrix_puppet_slack',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string,
     }] if matrix_mx_puppet_slack_enabled else [])
     + ([{
       'name': 'matrix_puppet_twitter',
       'username': 'matrix_puppet_twitter',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string,
     }] if matrix_mx_puppet_twitter_enabled else [])
     + ([{
       'name': 'matrix_puppet_instagram',
       'username': 'matrix_puppet_instagram',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string,
     }] if matrix_mx_puppet_instagram_enabled else [])
     + ([{
       'name': 'matrix_puppet_discord',
       'username': 'matrix_puppet_discord',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string,
     }] if matrix_mx_puppet_discord_enabled else [])
     + ([{
       'name': 'matrix_puppet_steam',
       'username': 'matrix_puppet_steam',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string,
     }] if matrix_mx_puppet_steam_enabled else [])
     + ([{
       'name': 'matrix_dimension',
       'username': 'matrix_dimension',
-      'pass': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string,
     }] if matrix_dimension_enabled else [])
   }}
 
diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 2645d4b0a34..ad417e0bd16 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -37,8 +37,8 @@ matrix_postgres_container_postgres_bind_port: ""
 # matrix_postgres_additional_databases:
 #   - name: matrix_appservice_discord
 #     username: matrix_appservice_discord
-#     pass: some_password
+#     password: some_password
 #   - name: matrix_appservice_slack
 #     username: matrix_appservice_slack
-#     pass: some_password
+#     password: some_password
 matrix_postgres_additional_databases: []
diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml
index a994cc260a4..ce064d598e2 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_database.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml
@@ -6,7 +6,7 @@
   fail:
     msg: "Additional database definition ({{ additional_db }} lacks a required key: {{ item }}"
   when: "item not in additional_db"
-  with_items: "{{ ['name', 'username', 'pass'] }}"
+  with_items: "{{ ['name', 'username', 'password'] }}"
 
 # The SQL statements that we'll run against Postgres are stored in a file that others can't read.
 # This file will be mounted into the container and fed to Postgres.
diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
index 732b01870cd..af3d912c5bb 100644
--- a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
+++ b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
@@ -10,7 +10,7 @@ $$;
 
 -- This is useful for initial user creation (since we don't assign a password above) and for handling subsequent password changes
 -- TODO - we should escape quotes in the password.
-ALTER ROLE {{ additional_db.username }} PASSWORD '{{ additional_db.pass }}';
+ALTER ROLE {{ additional_db.username }} PASSWORD '{{ additional_db.password }}';
 
 -- This will generate an error on subsequent execution
 CREATE DATABASE {{ additional_db.name }} WITH LC_CTYPE 'C' LC_COLLATE 'C' OWNER {{ additional_db.username }};

From a374d309c8f5d6038bfd5f94a66c3cd9a68c7e23 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 00:52:25 +0200
Subject: [PATCH 13/79] Make appservice-discord support both SQLite and
 Postgres

People can toggle between them now. The playbook also defaults
to using SQLite if an external Postgres server is used.

Ideally, we'd be able to create databases/users in external Postgres
servers as well, but our initialization logic (and `docker run` command,
etc.) hardcode too many things right now.
---
 group_vars/matrix_servers                        | 13 +++++++++----
 .../defaults/main.yml                            | 16 ++++++++++++++++
 .../tasks/validate_config.yml                    |  4 ++++
 .../templates/config.yaml.j2                     |  7 +++++--
 4 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 3169e18e46a..717e71bc67b 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -67,6 +67,11 @@ matrix_appservice_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_k
 
 matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'discord.hs.token') | to_uuid }}"
 
+# We only make this use Postgres if our own Postgres server is enabled.
+# It's only then (for now) that we can automatically create the necessary database and user for this service.
+matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') }}"
+
 ######################################################################
 #
 # /matrix-bridge-appservice-discord
@@ -892,10 +897,10 @@ matrix_postgres_db_name: "homeserver"
 matrix_postgres_additional_databases: |
   {{
     ([{
-      'name': 'matrix_appservice_discord',
-      'username': 'matrix_appservice_discord',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_discord.db.secret') | string,
-    }] if matrix_appservice_discord_enabled else [])
+      'name': matrix_appservice_discord_database_connString_db_name,
+      'username': matrix_appservice_discord_database_connString_username,
+      'password': matrix_appservice_discord_database_connString_password,
+    }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_connString_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_appservice_slack',
       'username': 'matrix_appservice_slack',
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 357b93d644f..36b39b8e364 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -41,6 +41,22 @@ matrix_appservice_discord_bridge_homeserverUrl: "http://matrix-synapse:8008"
 matrix_appservice_discord_bridge_disablePresence: false
 matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`)
+# - adjust your database credentials
+matrix_appservice_discord_database_engine: 'sqlite'
+matrix_appservice_discord_database_filename: "/data/discord.db"
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_connString_username }}:{{ matrix_appservice_discord_database_connString_password }}@{{ matrix_appservice_discord_database_connString_hostname }}:{{ matrix_appservice_discord_database_connString_port }}/{{ matrix_appservice_discord_database_connString_db_name }}'
+matrix_appservice_discord_database_connString_username: 'matrix_appservice_discord'
+matrix_appservice_discord_database_connString_password: 'some-password'
+matrix_appservice_discord_database_connString_hostname: 'matrix-postgres'
+matrix_appservice_discord_database_connString_port: 5432
+matrix_appservice_discord_database_connString_db_name: 'matrix_appservice_discord'
+
 # Tells whether the bot should make use of "Privileged Gateway Intents".
 #
 # Enabling this means that you need to enable it for the bot (Discord application) as well,
diff --git a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml
index b0713a43ddb..73253ba030b 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml
@@ -20,3 +20,7 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_appservice_discord_container_expose_client_server_api_port', 'new': '<superseded by matrix_appservice_discord_container_http_host_bind_port>'}
+
+- name: Require a valid database engine
+  fail: msg="`matrix_appservice_discord_database_engine` needs to be either 'sqlite' or 'postgres'"
+  when: "matrix_appservice_discord_database_engine not in ['sqlite', 'postgres']"
diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
index de4182d9a4a..b2ecd1981d3 100644
--- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2
@@ -58,8 +58,11 @@ database:
   # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite
   # WARNING: You will almost certainly be fine with sqlite unless your bridge
   # is in heavy demand and you suffer from IO slowness.
-  #filename: "/data/discord.db"
-  connString: "postgresql://matrix_appservice_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_appservice_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_discord"
+  {% if matrix_appservice_discord_database_engine == 'sqlite' %}
+  filename: {{ matrix_appservice_discord_database_filename|to_json }}
+  {% else %}
+  connString: {{ matrix_appservice_discord_database_connString|to_json }}
+  {% endif %}
 room:
   # Set the default visibility of alias rooms, defaults to "public".
   # One of: "public", "private"

From 183d2a10dbdfc142ace8e88e515e2d2f60ab84bc Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 00:59:59 +0200
Subject: [PATCH 14/79] Ensure matrix-postgres.service is started before
 creating additional users/databases

---
 roles/matrix-postgres/defaults/main.yml        |  6 ++++++
 .../tasks/util/create_additional_databases.yml | 18 ++++++++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index ad417e0bd16..89559f5d073 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -42,3 +42,9 @@ matrix_postgres_container_postgres_bind_port: ""
 #     username: matrix_appservice_slack
 #     password: some_password
 matrix_postgres_additional_databases: []
+
+# The number of seconds to wait after starting `matrix-postgres.service`
+# and before trying to run queries for creating additional databases/users against it.
+#
+# For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all.
+matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15
diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
index 51deb228163..0ad460dddb9 100644
--- a/roles/matrix-postgres/tasks/util/create_additional_databases.yml
+++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml
@@ -1,10 +1,20 @@
 ---
 
-# TODO - we should ensure matrix-postgres.service is started.
-# .. and that if we had just started it, we've given it ample time to initialize,
-# before we attempt to run queries against it.
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
 
-- name: Create additional Postgers user and database
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+- name: Create additional Postgres user and database
   include_tasks: "{{ role_path }}/tasks/util/create_additional_database.yml"
   with_items: "{{ matrix_postgres_additional_databases }}"
   loop_control:

From dd994995bca0cb0bd162aec1f5ee4dae35848614 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 01:22:23 +0200
Subject: [PATCH 15/79] Simplify password for additional Postgres databases

Using the result of `password_hash` works for creating them,
but authentication seems to be failing with some tools like pgloader.

It's possible that we're not escaping things properly somewhere.
Ideally, it'd be nice to solve that. But the easier (and still
relatively safe/good) solution is to just turn that password hash
into a UUID that's safe for passing around without worrying about
escaping.
---
 group_vars/matrix_servers | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 717e71bc67b..b9d0be26439 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -70,7 +70,7 @@ matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_k
 # We only make this use Postgres if our own Postgres server is enabled.
 # It's only then (for now) that we can automatically create the necessary database and user for this service.
 matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') }}"
+matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -904,72 +904,72 @@ matrix_postgres_additional_databases: |
     + ([{
       'name': 'matrix_appservice_slack',
       'username': 'matrix_appservice_slack',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | to_uuid,
     }] if matrix_appservice_slack_enabled else [])
     + ([{
       'name': 'matrix_appservice_irc',
       'username': 'matrix_appservice_irc',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | to_uuid,
     }] if matrix_appservice_irc_enabled else [])
     + ([{
       'name': 'mautrix_bridge_facebook',
       'username': 'mautrix_bridge_facebook',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | to_uuid,
     }] if matrix_mautrix_facebook_enabled else [])
     + ([{
       'name': 'mautrix_bridge_hangouts',
       'username': 'mautrix_bridge_hangouts',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | to_uuid,
     }] if matrix_mautrix_hangouts_enabled else [])
     + ([{
       'name': 'mautrix_bridge_telegram',
       'username': 'mautrix_bridge_telegram',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | to_uuid,
     }] if matrix_mautrix_telegram_enabled else [])
     + ([{
       'name': 'mautrix_bridge_whatsapp',
       'username': 'mautrix_bridge_whatsapp',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | to_uuid,
     }] if matrix_mautrix_whatsapp_enabled else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
       'name': 'matrix_puppet_skype',
       'username': 'matrix_puppet_skype',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | to_uuid,
     }] if matrix_mx_puppet_skype_enabled else [])
     + ([{
       'name': 'matrix_puppet_slack',
       'username': 'matrix_puppet_slack',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | to_uuid,
     }] if matrix_mx_puppet_slack_enabled else [])
     + ([{
       'name': 'matrix_puppet_twitter',
       'username': 'matrix_puppet_twitter',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | to_uuid,
     }] if matrix_mx_puppet_twitter_enabled else [])
     + ([{
       'name': 'matrix_puppet_instagram',
       'username': 'matrix_puppet_instagram',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | to_uuid,
     }] if matrix_mx_puppet_instagram_enabled else [])
     + ([{
       'name': 'matrix_puppet_discord',
       'username': 'matrix_puppet_discord',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | to_uuid,
     }] if matrix_mx_puppet_discord_enabled else [])
     + ([{
       'name': 'matrix_puppet_steam',
       'username': 'matrix_puppet_steam',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | to_uuid,
     }] if matrix_mx_puppet_steam_enabled else [])
     + ([{
       'name': 'matrix_dimension',
       'username': 'matrix_dimension',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | string,
+      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | to_uuid,
     }] if matrix_dimension_enabled else [])
   }}
 

From b87b75437232a8b3fa4c2ba2258382dfa761a62e Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 01:36:15 +0200
Subject: [PATCH 16/79] Fail if appservice-discord wants Postgres, but has
 leftover SQLite data

---
 .../defaults/main.yml                            |  3 ++-
 .../tasks/setup_install.yml                      | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 36b39b8e364..53d7968d426 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -49,7 +49,8 @@ matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 # - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`)
 # - adjust your database credentials
 matrix_appservice_discord_database_engine: 'sqlite'
-matrix_appservice_discord_database_filename: "/data/discord.db"
+matrix_appservice_discord_database_filename: "/data/{{ matrix_appservice_discord_database_filename_name }}"
+matrix_appservice_discord_database_filename_name: "discord.db"
 matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_connString_username }}:{{ matrix_appservice_discord_database_connString_password }}@{{ matrix_appservice_discord_database_connString_hostname }}:{{ matrix_appservice_discord_database_connString_port }}/{{ matrix_appservice_discord_database_connString_db_name }}'
 matrix_appservice_discord_database_connString_username: 'matrix_appservice_discord'
 matrix_appservice_discord_database_connString_password: 'some-password'
diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index 935d07ebfeb..cea8fda8121 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -1,5 +1,21 @@
 ---
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}"
+    register: matrix_appservice_discord_stat_sqlite_db
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file.
+        To migrate to Postgres: TODO - migration instructions here.
+  when: "matrix_appservice_discord_database_engine == 'postgres'"
+
 - name: Ensure Appservice Discord image is pulled
   docker_image:
     name: "{{ matrix_appservice_discord_docker_image }}"

From cb969c6ca297b21216f896033ae86702be227812 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 02:23:29 +0200
Subject: [PATCH 17/79] Add --tags=import-generic-sqlite-db (pgloader import)

This can be used by various bridges, etc., to import an SQLite
(or some other supported) database into Postgres.
---
 roles/matrix-postgres/defaults/main.yml       |  2 +
 .../tasks/import_generic_sqlite_db.yml        | 84 +++++++++++++++++++
 roles/matrix-postgres/tasks/main.yml          |  6 ++
 3 files changed, 92 insertions(+)
 create mode 100644 roles/matrix-postgres/tasks/import_generic_sqlite_db.yml

diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 89559f5d073..5755742f58a 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -48,3 +48,5 @@ matrix_postgres_additional_databases: []
 #
 # For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all.
 matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15
+
+matrix_postgres_pgloader_docker_image: "docker.io/illagrenan/pgloader:3.6.2"
diff --git a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml
new file mode 100644
index 00000000000..f2798a73d64
--- /dev/null
+++ b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml
@@ -0,0 +1,84 @@
+---
+
+# Pre-checks
+
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import."
+  when: "not matrix_postgres_enabled|bool"
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: "The `sqlite_database_path` variable needs to be provided to this playbook, via --extra-vars"
+  when: "sqlite_database_path is not defined or sqlite_database_path.startswith('<')"
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: >-
+      The `postgres_db_connection_string` variable needs to be provided to this playbook, via `--extra-vars`.
+      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name`"
+  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
+
+- name: Check if the provided SQLite database file exists
+  stat:
+    path: "{{ sqlite_database_path }}"
+  register: sqlite_database_path_stat_result
+
+- name: Fail if provided SQLite database file doesn't exist
+  fail:
+    msg: "File cannot be found on the server at {{ sqlite_database_path }}"
+  when: "not sqlite_database_path_stat_result.stat.exists"
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+- name: Import SQLite database from {{ sqlite_database_path }} into Postgres
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ sqlite_database_path }},dst=/in.db,ro
+      --entrypoint=/bin/sh
+      {{ matrix_postgres_pgloader_docker_image }}
+      -c
+      'pgloader /in.db {{ postgres_db_connection_string }}'
+
+- name: Archive SQLite database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup)
+  command:
+    cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup"
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your SQLite database file has been imported into Postgres. The original file has been moved from `{{ sqlite_database_path }}` to `{{ sqlite_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
+        ]
+      }}
diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml
index f4c752a0401..b9c2ae7c99c 100644
--- a/roles/matrix-postgres/tasks/main.yml
+++ b/roles/matrix-postgres/tasks/main.yml
@@ -26,6 +26,12 @@
   tags:
     - import-synapse-sqlite-db
 
+# Perhaps we need a new variable here, instead of `run_postgres_import_sqlite_db`.
+- import_tasks: "{{ role_path }}/tasks/import_generic_sqlite_db.yml"
+  when: run_postgres_import_sqlite_db|bool
+  tags:
+    - import-generic-sqlite-db
+
 - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
   when: run_postgres_upgrade|bool
   tags:

From 4617984b9f998e032f426ec629e408a1f0c2ecf8 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 02:24:32 +0200
Subject: [PATCH 18/79] Add (SQLite -> Postgres) migration instructions

---
 .../tasks/setup_install.yml                       | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index cea8fda8121..57e59df06a9 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -1,10 +1,13 @@
 ---
 
 - block:
+  - set_fact:
+      matrix_appservice_discord_sqlite_db_path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}"
+
   - name: Check if an SQLite database already exists
     stat:
-      path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}"
-    register: matrix_appservice_discord_stat_sqlite_db
+      path: "{{ matrix_appservice_discord_sqlite_db_path }}"
+    register: matrix_appservice_discord_sqlite_db_path_stat_result
 
   - name: Fail if an SQLite database already exists when using Postgres
     fail:
@@ -12,8 +15,12 @@
         matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now).
         However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}.
         It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file.
-        To migrate to Postgres: TODO - migration instructions here.
+        To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_db_path }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_appservice_discord_sqlite_db_path_stat_result.stat.exists"
   when: "matrix_appservice_discord_database_engine == 'postgres'"
 
 - name: Ensure Appservice Discord image is pulled

From b217a2ed6907347200c57d66195dd780e4a1f24f Mon Sep 17 00:00:00 2001
From: transcaffeine <transcaffeine@finallycoffee.eu>
Date: Mon, 14 Dec 2020 07:54:24 +0100
Subject: [PATCH 19/79] postgres: set default in playbook to postgres but
 fallback to sqlite

---
 group_vars/matrix_servers | 175 +++++++++++++++++++++++++-------------
 1 file changed, 118 insertions(+), 57 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index b9d0be26439..0cbc4587980 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -140,6 +140,10 @@ matrix_appservice_slack_systemd_required_services_list: |
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
   }}
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-appservice-slack
@@ -175,6 +179,11 @@ matrix_appservice_irc_appservice_token: "{{ matrix_synapse_macaroon_secret_key |
 
 matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'irc.hs.token') | to_uuid }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
+
+
 ######################################################################
 #
 # /matrix-bridge-appservice-irc
@@ -208,6 +217,10 @@ matrix_mautrix_facebook_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matrix_synapse_enabled else true }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mautrix-facebook
@@ -241,6 +254,10 @@ matrix_mautrix_hangouts_container_http_host_bind_port: "{{ '' if matrix_nginx_pr
 
 matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mautrix-hangouts
@@ -277,6 +294,10 @@ matrix_mautrix_telegram_container_http_host_bind_port: "{{ '' if matrix_nginx_pr
 
 matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mautrix-telegram
@@ -305,6 +326,10 @@ matrix_mautrix_whatsapp_homeserver_token: "{{ matrix_synapse_macaroon_secret_key
 
 matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mautrix-whatsapp
@@ -361,6 +386,10 @@ matrix_mx_puppet_skype_homeserver_token: "{{ matrix_synapse_macaroon_secret_key
 
 matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-skype
@@ -392,6 +421,10 @@ matrix_mx_puppet_slack_homeserver_token: "{{ matrix_synapse_macaroon_secret_key
 
 matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-slack
@@ -424,6 +457,10 @@ matrix_mx_puppet_twitter_login_shared_secret: "{{ matrix_synapse_ext_password_pr
 
 matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else ('127.0.0.1:' ~ matrix_mx_puppet_twitter_appservice_port) }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-twitter
@@ -455,6 +492,10 @@ matrix_mx_puppet_instagram_homeserver_token: "{{ matrix_synapse_macaroon_secret_
 
 matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-instagram
@@ -485,6 +526,10 @@ matrix_mx_puppet_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_ke
 
 matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-discord
@@ -515,6 +560,10 @@ matrix_mx_puppet_steam_homeserver_token: "{{ matrix_synapse_macaroon_secret_key
 
 matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bridge-mx-puppet-steam
@@ -625,6 +674,10 @@ matrix_dimension_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_ena
 matrix_integration_manager_rest_url: "{{ matrix_dimension_integrations_rest_url if matrix_dimension_enabled else None }}"
 matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if matrix_dimension_enabled else None }}"
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-dimension
@@ -902,75 +955,83 @@ matrix_postgres_additional_databases: |
       'password': matrix_appservice_discord_database_connString_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_connString_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': 'matrix_appservice_slack',
-      'username': 'matrix_appservice_slack',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_slack.db.secret') | to_uuid,
-    }] if matrix_appservice_slack_enabled else [])
+      'name': matrix_appservice_slack_database_db_name
+      'username': matrix_appservice_slack_database_username
+      'password': matrix_appservice_slack_database_password
+    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'matrix_appservice_irc',
-      'username': 'matrix_appservice_irc',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'app_irc.db.secret') | to_uuid,
-    }] if matrix_appservice_irc_enabled else [])
+      'name': matrix_appservice_irc_database_db_name
+      'username': matrix_appservice_irc_database_username
+      'password': matrix_appservice_irc_database_password
+    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'mautrix_bridge_facebook',
-      'username': 'mautrix_bridge_facebook',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_facebook.db.secret') | to_uuid,
-    }] if matrix_mautrix_facebook_enabled else [])
+      'name': matrix_mautrix_facebook_database_db_name
+      'username': matrix_mautrix_facebook_database_username
+      'password': matrix_mautrix_facebook_database_password
+    }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'mautrix_bridge_hangouts',
-      'username': 'mautrix_bridge_hangouts',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_hangouts.db.secret') | to_uuid,
-    }] if matrix_mautrix_hangouts_enabled else [])
+      'name': matrix_mautrix_hangouts_database_db_name
+      'username': matrix_mautrix_hangouts_database_username
+      'password': matrix_mautrix_hangouts_database_password
+    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'mautrix_bridge_telegram',
-      'username': 'mautrix_bridge_telegram',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_telegram.db.secret') | to_uuid,
-    }] if matrix_mautrix_telegram_enabled else [])
+      'name': matrix_mautrix_telegram_database_db_name
+      'username': matrix_mautrix_telegram_database_username
+      'password': matrix_mautrix_telegram_database_password
+    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'mautrix_bridge_whatsapp',
-      'username': 'mautrix_bridge_whatsapp',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mautrix_whatsapp.db.secret') | to_uuid,
-    }] if matrix_mautrix_whatsapp_enabled else [])
+      'name': matrix_mautrix_whatsapp_database_db_name
+      'username': matrix_mautrix_whatsapp_database_username
+      'password': matrix_mautrix_whatsapp_database_password
+    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres' else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      'name': 'matrix_puppet_skype',
-      'username': 'matrix_puppet_skype',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_skype.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_skype_enabled else [])
-    + ([{
-      'name': 'matrix_puppet_slack',
-      'username': 'matrix_puppet_slack',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_slack.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_slack_enabled else [])
-    + ([{
-      'name': 'matrix_puppet_twitter',
-      'username': 'matrix_puppet_twitter',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_twitter.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_twitter_enabled else [])
-    + ([{
-      'name': 'matrix_puppet_instagram',
-      'username': 'matrix_puppet_instagram',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_insta.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_instagram_enabled else [])
-    + ([{
-      'name': 'matrix_puppet_discord',
-      'username': 'matrix_puppet_discord',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_puppet.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_discord_enabled else [])
-    + ([{
-      'name': 'matrix_puppet_steam',
-      'username': 'matrix_puppet_steam',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx_steam.db.secret') | to_uuid,
-    }] if matrix_mx_puppet_steam_enabled else [])
+      'name': matrix_mx_puppet_skype_database_db_name
+      'username': matrix_mx_puppet_skype_database_username
+      'password': matrix_mx_puppet_skype_database_password
+    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres' else [])
     + ([{
-      'name': 'matrix_dimension',
-      'username': 'matrix_dimension',
-      'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db.secret') | to_uuid,
-    }] if matrix_dimension_enabled else [])
+      'name': matrix_mx_puppet_slack_database_db_name
+      'username': matrix_mx_puppet_slack_database_username
+      'password': matrix_mx_puppet_slack_database_password
+    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres' else [])
+     + ([{
+      'name': matrix_mx_puppet_twitter_database_db_name
+      'username': matrix_mx_puppet_twitter_database_username
+      'password': matrix_mx_puppet_twitter_database_password
+    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres' else [])
+     + ([{
+      'name': matrix_mx_puppet_instagram_database_db_name
+      'username': matrix_mx_puppet_instagram_database_username
+      'password': matrix_mx_puppet_instagram_database_password
+    }] if (matrix_mx_puppet_instagram_enabled
+           and matrix_mx_puppet_instagram_database_engine == 'postgres'
+           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres' else [])
+     + ([{
+      'name': matrix_mx_puppet_discord_database_db_name
+      'username': matrix_mx_puppet_discord_database_username
+      'password': matrix_mx_puppet_discord_database_password
+    }] if (matrix_mx_puppet_discord_enabled
+           and matrix_mx_puppet_discord_database_engine == 'postgres'
+           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres' else [])
+     + ([{
+      'name': matrix_mx_puppet_steam_database_db_name
+      'username': matrix_mx_puppet_steam_database_username
+      'password': matrix_mx_puppet_steam_database_password
+    }] if (matrix_mx_puppet_steam_enabled
+           and matrix_mx_puppet_steam_database_engine == 'postgres'
+           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres' else [])
+     + ([{
+      'name': matrix_dimension_database_db_name
+      'username': matrix_dimension_database_username
+      'password': matrix_dimension_database_password
+    }] if (matrix_dimension_enabled
+           and matrix_dimension_database_engine == 'postgres'
+           and matrix_dimension_database_hostname == 'matrix-postgres' else [])
   }}
 
 ######################################################################

From f1e85f7112ffa9dd3309e8963c9a8e69723f48a0 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 10:04:37 +0200
Subject: [PATCH 20/79] Don't mention Postgres roles, just say users

---
 .../templates/init-additional-db-user-and-role.sql.j2           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
index af3d912c5bb..609a1344824 100644
--- a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
+++ b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2
@@ -4,7 +4,7 @@ DO $$
 BEGIN
   CREATE USER {{ additional_db.username }};
   EXCEPTION WHEN DUPLICATE_OBJECT THEN
-  RAISE NOTICE 'not creating role {{ additional_db.username }}, since it already exists';
+  RAISE NOTICE 'not creating user {{ additional_db.username }}, since it already exists';
 END
 $$;
 

From d91aa5a060ab0742f25c15a951429104c1793a91 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 10:51:53 +0200
Subject: [PATCH 21/79] Do not introduce sub-variables exposing implementation
 details

---
 group_vars/matrix_servers                     | 10 ++++----
 .../defaults/main.yml                         | 25 ++++++++++++-------
 .../tasks/setup_install.yml                   | 13 ++++------
 3 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 0cbc4587980..efb29d4a2de 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -70,7 +70,7 @@ matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_k
 # We only make this use Postgres if our own Postgres server is enabled.
 # It's only then (for now) that we can automatically create the necessary database and user for this service.
 matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_discord_database_connString_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}"
+matrix_appservice_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -950,10 +950,10 @@ matrix_postgres_db_name: "homeserver"
 matrix_postgres_additional_databases: |
   {{
     ([{
-      'name': matrix_appservice_discord_database_connString_db_name,
-      'username': matrix_appservice_discord_database_connString_username,
-      'password': matrix_appservice_discord_database_connString_password,
-    }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_connString_hostname == 'matrix-postgres') else [])
+      'name': matrix_appservice_discord_postgres_db_name,
+      'username': matrix_appservice_discord_postgres_username,
+      'password': matrix_appservice_discord_postgres_password,
+    }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_appservice_slack_database_db_name
       'username': matrix_appservice_slack_database_username
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 53d7968d426..fb517cd5f6e 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -47,16 +47,23 @@ matrix_appservice_discord_bridge_enableSelfServiceBridging: false
 #
 # To use Postgres:
 # - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`)
-# - adjust your database credentials
+# - adjust your database credentials via the `matrix_appservice_discord_postgres_*` variables
 matrix_appservice_discord_database_engine: 'sqlite'
-matrix_appservice_discord_database_filename: "/data/{{ matrix_appservice_discord_database_filename_name }}"
-matrix_appservice_discord_database_filename_name: "discord.db"
-matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_connString_username }}:{{ matrix_appservice_discord_database_connString_password }}@{{ matrix_appservice_discord_database_connString_hostname }}:{{ matrix_appservice_discord_database_connString_port }}/{{ matrix_appservice_discord_database_connString_db_name }}'
-matrix_appservice_discord_database_connString_username: 'matrix_appservice_discord'
-matrix_appservice_discord_database_connString_password: 'some-password'
-matrix_appservice_discord_database_connString_hostname: 'matrix-postgres'
-matrix_appservice_discord_database_connString_port: 5432
-matrix_appservice_discord_database_connString_db_name: 'matrix_appservice_discord'
+
+matrix_appservice_discord_sqlite_database_path_local: "{{ matrix_appservice_discord_data_path }}/discord.db"
+matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db"
+
+matrix_appservice_discord_postgres_username: 'matrix_appservice_discord'
+matrix_appservice_discord_postgres_password: 'some-password'
+matrix_appservice_discord_postgres_hostname: 'matrix-postgres'
+matrix_appservice_discord_postgres_port: 5432
+matrix_appservice_discord_postgres_db_name: 'matrix_appservice_discord'
+
+# These 2 variables are what actually ends up in the bridge configuration.
+# It's best if you don't change them directly, but rather redefine the sub-variables that constitute them.
+matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}"
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_postgres_username }}:{{ matrix_appservice_discord_postgres_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_postgres_port }}/{{ matrix_appservice_discord_postgres_db_name }}'
+
 
 # Tells whether the bot should make use of "Privileged Gateway Intents".
 #
diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index 57e59df06a9..86a5fe517e7 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -1,26 +1,23 @@
 ---
 
 - block:
-  - set_fact:
-      matrix_appservice_discord_sqlite_db_path: "{{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}"
-
   - name: Check if an SQLite database already exists
     stat:
-      path: "{{ matrix_appservice_discord_sqlite_db_path }}"
-    register: matrix_appservice_discord_sqlite_db_path_stat_result
+      path: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
+    register: matrix_appservice_discord_sqlite_database_path_local_stat_result
 
   - name: Fail if an SQLite database already exists when using Postgres
     fail:
       msg: >-
         matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_data_path }}/{{ matrix_appservice_discord_database_filename_name }}.
+        However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_sqlite_database_path_local }}.
         It appears that you've been using this bridge with the SQLite engine until now.
         To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_db_path }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_appservice_discord_sqlite_db_path_stat_result.stat.exists"
+    when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_appservice_discord_database_engine == 'postgres'"
 
 - name: Ensure Appservice Discord image is pulled

From c0edacbefd60d5bfd136c448ff3fdf76da645731 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 11:17:12 +0200
Subject: [PATCH 22/79] Fix matrix_postgres_additional_databases syntax
 problems

---
 group_vars/matrix_servers | 106 +++++++++++++++++++-------------------
 1 file changed, 53 insertions(+), 53 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index efb29d4a2de..763fccd862e 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -955,84 +955,84 @@ matrix_postgres_additional_databases: |
       'password': matrix_appservice_discord_postgres_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_slack_database_db_name
-      'username': matrix_appservice_slack_database_username
-      'password': matrix_appservice_slack_database_password
-    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_appservice_slack_database_db_name,
+      'username': matrix_appservice_slack_database_username,
+      'password': matrix_appservice_slack_database_password,
+    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_irc_database_db_name
-      'username': matrix_appservice_irc_database_username
-      'password': matrix_appservice_irc_database_password
-    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_appservice_irc_database_db_name,
+      'username': matrix_appservice_irc_database_username,
+      'password': matrix_appservice_irc_database_password,
+    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_facebook_database_db_name
-      'username': matrix_mautrix_facebook_database_username
-      'password': matrix_mautrix_facebook_database_password
-    }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mautrix_facebook_postgres_db_name,
+      'username': matrix_mautrix_facebook_postgres_username,
+      'password': matrix_mautrix_facebook_postgres_password,
+    }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_hangouts_database_db_name
-      'username': matrix_mautrix_hangouts_database_username
-      'password': matrix_mautrix_hangouts_database_password
-    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mautrix_hangouts_database_db_name,
+      'username': matrix_mautrix_hangouts_database_username,
+      'password': matrix_mautrix_hangouts_database_password,
+    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_telegram_database_db_name
-      'username': matrix_mautrix_telegram_database_username
-      'password': matrix_mautrix_telegram_database_password
-    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mautrix_telegram_database_db_name,
+      'username': matrix_mautrix_telegram_database_username,
+      'password': matrix_mautrix_telegram_database_password,
+    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_whatsapp_database_db_name
-      'username': matrix_mautrix_whatsapp_database_username
-      'password': matrix_mautrix_whatsapp_database_password
-    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mautrix_whatsapp_database_db_name,
+      'username': matrix_mautrix_whatsapp_database_username,
+      'password': matrix_mautrix_whatsapp_database_password,
+    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      'name': matrix_mx_puppet_skype_database_db_name
-      'username': matrix_mx_puppet_skype_database_username
-      'password': matrix_mx_puppet_skype_database_password
-    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mx_puppet_skype_database_db_name,
+      'username': matrix_mx_puppet_skype_database_username,
+      'password': matrix_mx_puppet_skype_database_password,
+    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mx_puppet_slack_database_db_name
-      'username': matrix_mx_puppet_slack_database_username
-      'password': matrix_mx_puppet_slack_database_password
-    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mx_puppet_slack_database_db_name,
+      'username': matrix_mx_puppet_slack_database_username,
+      'password': matrix_mx_puppet_slack_database_password,
+    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_twitter_database_db_name
-      'username': matrix_mx_puppet_twitter_database_username
-      'password': matrix_mx_puppet_twitter_database_password
-    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres' else [])
+      'name': matrix_mx_puppet_twitter_database_db_name,
+      'username': matrix_mx_puppet_twitter_database_username,
+      'password': matrix_mx_puppet_twitter_database_password,
+    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_instagram_database_db_name
-      'username': matrix_mx_puppet_instagram_database_username
-      'password': matrix_mx_puppet_instagram_database_password
+      'name': matrix_mx_puppet_instagram_database_db_name,
+      'username': matrix_mx_puppet_instagram_database_username,
+      'password': matrix_mx_puppet_instagram_database_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
-           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres' else [])
+           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_discord_database_db_name
-      'username': matrix_mx_puppet_discord_database_username
-      'password': matrix_mx_puppet_discord_database_password
+      'name': matrix_mx_puppet_discord_database_db_name,
+      'username': matrix_mx_puppet_discord_database_username,
+      'password': matrix_mx_puppet_discord_database_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
-           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres' else [])
+           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_steam_database_db_name
-      'username': matrix_mx_puppet_steam_database_username
-      'password': matrix_mx_puppet_steam_database_password
+      'name': matrix_mx_puppet_steam_database_db_name,
+      'username': matrix_mx_puppet_steam_database_username,
+      'password': matrix_mx_puppet_steam_database_password,
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
-           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres' else [])
+           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_dimension_database_db_name
-      'username': matrix_dimension_database_username
-      'password': matrix_dimension_database_password
+      'name': matrix_dimension_database_db_name,
+      'username': matrix_dimension_database_username,
+      'password': matrix_dimension_database_password,
     }] if (matrix_dimension_enabled
            and matrix_dimension_database_engine == 'postgres'
-           and matrix_dimension_database_hostname == 'matrix-postgres' else [])
-  }}
+           and matrix_dimension_database_hostname == 'matrix-postgres') else [])
+   }}
 
 ######################################################################
 #

From 2a99e84b5bfd9f3988566e498b2fe729baa81c75 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 11:19:18 +0200
Subject: [PATCH 23/79] Rename Postgres-related variables (`matrix*database_*`
 -> `matrix*postgres_*`)

`matrix_SERVICE_database_engine` remains as it is - the only one with a
`_database_` infix. Postgres-related ones move to `_postgres_`.
---
 group_vars/matrix_servers | 72 +++++++++++++++++++--------------------
 1 file changed, 36 insertions(+), 36 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 763fccd862e..5e3695cac51 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -955,14 +955,14 @@ matrix_postgres_additional_databases: |
       'password': matrix_appservice_discord_postgres_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_slack_database_db_name,
-      'username': matrix_appservice_slack_database_username,
-      'password': matrix_appservice_slack_database_password,
+      'name': matrix_appservice_slack_postgres_db_name,
+      'username': matrix_appservice_slack_postgres_username,
+      'password': matrix_appservice_slack_postgres_password,
     }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_irc_database_db_name,
-      'username': matrix_appservice_irc_database_username,
-      'password': matrix_appservice_irc_database_password,
+      'name': matrix_appservice_irc_postgres_db_name,
+      'username': matrix_appservice_irc_postgres_username,
+      'password': matrix_appservice_irc_postgres_password,
     }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_facebook_postgres_db_name,
@@ -970,19 +970,19 @@ matrix_postgres_additional_databases: |
       'password': matrix_mautrix_facebook_postgres_password,
     }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_hangouts_database_db_name,
-      'username': matrix_mautrix_hangouts_database_username,
-      'password': matrix_mautrix_hangouts_database_password,
+      'name': matrix_mautrix_hangouts_postgres_db_name,
+      'username': matrix_mautrix_hangouts_postgres_username,
+      'password': matrix_mautrix_hangouts_postgres_password,
     }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_telegram_database_db_name,
-      'username': matrix_mautrix_telegram_database_username,
-      'password': matrix_mautrix_telegram_database_password,
+      'name': matrix_mautrix_telegram_postgres_db_name,
+      'username': matrix_mautrix_telegram_postgres_username,
+      'password': matrix_mautrix_telegram_postgres_password,
     }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_whatsapp_database_db_name,
-      'username': matrix_mautrix_whatsapp_database_username,
-      'password': matrix_mautrix_whatsapp_database_password,
+      'name': matrix_mautrix_whatsapp_postgres_db_name,
+      'username': matrix_mautrix_whatsapp_postgres_username,
+      'password': matrix_mautrix_whatsapp_postgres_password,
     }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_bridge_sms',
@@ -990,45 +990,45 @@ matrix_postgres_additional_databases: |
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      'name': matrix_mx_puppet_skype_database_db_name,
-      'username': matrix_mx_puppet_skype_database_username,
-      'password': matrix_mx_puppet_skype_database_password,
+      'name': matrix_mx_puppet_skype_postgres_db_name,
+      'username': matrix_mx_puppet_skype_postgres_username,
+      'password': matrix_mx_puppet_skype_postgres_password,
     }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mx_puppet_slack_database_db_name,
-      'username': matrix_mx_puppet_slack_database_username,
-      'password': matrix_mx_puppet_slack_database_password,
+      'name': matrix_mx_puppet_slack_postgres_db_name,
+      'username': matrix_mx_puppet_slack_postgres_username,
+      'password': matrix_mx_puppet_slack_postgres_password,
     }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_twitter_database_db_name,
-      'username': matrix_mx_puppet_twitter_database_username,
-      'password': matrix_mx_puppet_twitter_database_password,
+      'name': matrix_mx_puppet_twitter_postgres_db_name,
+      'username': matrix_mx_puppet_twitter_postgres_username,
+      'password': matrix_mx_puppet_twitter_postgres_password,
     }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_instagram_database_db_name,
-      'username': matrix_mx_puppet_instagram_database_username,
-      'password': matrix_mx_puppet_instagram_database_password,
+      'name': matrix_mx_puppet_instagram_postgres_db_name,
+      'username': matrix_mx_puppet_instagram_postgres_username,
+      'password': matrix_mx_puppet_instagram_postgres_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
            and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_discord_database_db_name,
-      'username': matrix_mx_puppet_discord_database_username,
-      'password': matrix_mx_puppet_discord_database_password,
+      'name': matrix_mx_puppet_discord_postgres_db_name,
+      'username': matrix_mx_puppet_discord_postgres_username,
+      'password': matrix_mx_puppet_discord_postgres_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
            and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_steam_database_db_name,
-      'username': matrix_mx_puppet_steam_database_username,
-      'password': matrix_mx_puppet_steam_database_password,
+      'name': matrix_mx_puppet_steam_postgres_db_name,
+      'username': matrix_mx_puppet_steam_postgres_username,
+      'password': matrix_mx_puppet_steam_postgres_password,
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
            and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_dimension_database_db_name,
-      'username': matrix_dimension_database_username,
-      'password': matrix_dimension_database_password,
+      'name': matrix_dimension_postgres_db_name,
+      'username': matrix_dimension_postgres_username,
+      'password': matrix_dimension_postgres_password,
     }] if (matrix_dimension_enabled
            and matrix_dimension_database_engine == 'postgres'
            and matrix_dimension_database_hostname == 'matrix-postgres') else [])

From 5dba0c038b6c19a9b430210cfaaa7538e383cef9 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 11:47:00 +0200
Subject: [PATCH 24/79] Make --tags=import-generic-sqlite-db commands not pass
 a sensitive connection string around

Instead of passing the connection string, we can now pass a name of a
variable, which contains a connection string.

Both are supported for having extra flexibility.
---
 .../tasks/setup_install.yml                   |  2 +-
 .../tasks/import_generic_sqlite_db.yml        | 27 ++++++++++++++-----
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index 86a5fe517e7..a9d8da4c402 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -15,7 +15,7 @@
         To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_db_connection_string={{ matrix_appservice_discord_database_connString }}'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_appservice_discord_database_connString'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_appservice_discord_database_engine == 'postgres'"
diff --git a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml
index f2798a73d64..a42c6f552ce 100644
--- a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml
+++ b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml
@@ -12,13 +12,6 @@
     msg: "The `sqlite_database_path` variable needs to be provided to this playbook, via --extra-vars"
   when: "sqlite_database_path is not defined or sqlite_database_path.startswith('<')"
 
-- name: Fail if playbook called incorrectly
-  fail:
-    msg: >-
-      The `postgres_db_connection_string` variable needs to be provided to this playbook, via `--extra-vars`.
-      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name`"
-  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
-
 - name: Check if the provided SQLite database file exists
   stat:
     path: "{{ sqlite_database_path }}"
@@ -29,6 +22,26 @@
     msg: "File cannot be found on the server at {{ sqlite_database_path }}"
   when: "not sqlite_database_path_stat_result.stat.exists"
 
+# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string,
+# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string.
+
+- block:
+    - name: Fail if postgres_connection_string_variable_name points to an undefined variable
+      fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`"
+      when: "postgres_connection_string_variable_name not in vars"
+
+    - name: Get Postgres connection string from variable
+      set_fact:
+        postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}"
+  when: 'postgres_connection_string_variable_name is defined'
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: >-
+      Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`.
+      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"`
+  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
+
 
 # Defaults
 

From 2d99ade72f09c9f25b718f0cda690319d9bea526 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 11:50:42 +0200
Subject: [PATCH 25/79] Add (Postgres + SQLite) support to
 matrix-mautrix-facebook bridge

---
 .../defaults/main.yml                         | 29 +++++++++++++++++++
 .../tasks/setup_install.yml                   | 21 ++++++++++++++
 .../templates/config.yaml.j2                  |  2 +-
 .../matrix-mautrix-facebook.service.j2        |  1 +
 4 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index 1547ad71f56..95adf354004 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -32,6 +32,35 @@ matrix_mautrix_facebook_systemd_wanted_services_list: []
 matrix_mautrix_facebook_appservice_token: ''
 matrix_mautrix_facebook_homeserver_token: ''
 
+
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_facebook_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_facebook_postgres_*` variables
+matrix_mautrix_facebook_database_engine: 'sqlite'
+
+matrix_mautrix_facebook_sqlite_database_path_local: "{{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db"
+matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebook.db"
+
+matrix_mautrix_facebook_postgres_username: 'matrix_mautrix_facebook'
+matrix_mautrix_facebook_postgres_password: 'some-password'
+matrix_mautrix_facebook_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_facebook_postgres_port: 5432
+matrix_mautrix_facebook_postgres_db_name: 'matrix_mautrix_facebook'
+
+matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_postgres_username }}:{{ matrix_mautrix_facebook_postgres_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_postgres_port }}/{{ matrix_mautrix_facebook_postgres_db_name }}'
+
+matrix_mautrix_facebook_appservice_database: "{{
+	{
+		'sqlite': matrix_mautrix_facebook_sqlite_database_path_in_container,
+		'postgres': matrix_mautrix_facebook_postgres_connection_string,
+	}[matrix_mautrix_facebook_database_engine]
+}}"
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_facebook_login_shared_secret: ''
 
diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
index 638854e2c5b..ef814108862 100644
--- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
@@ -1,5 +1,26 @@
 ---
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}"
+    register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_facebook_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_facebook_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_facebook_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_facebook_database_engine == 'postgres'"
+
+
 # If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist.
 # We don't want to fail in such cases.
 - name: Fail if matrix-synapse role already executed
diff --git a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
index 304571c136e..09287362ff9 100644
--- a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: sqlite://matrix_bridge_facebook:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_facebook') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_facebook
+    database: {{ matrix_mautrix_facebook_appservice_database|to_json }}
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     public:
diff --git a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2
index 1818465099a..caa52eb570b 100644
--- a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2
+++ b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2
@@ -18,6 +18,7 @@ ExecStartPre={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-fac
 			--log-driver=none \
 			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
 			--cap-drop=ALL \
+			--network={{ matrix_docker_network }} \
 			-v {{ matrix_mautrix_facebook_config_path }}:/config:z \
 			-v {{ matrix_mautrix_facebook_data_path }}:/data:z \
 			{{ matrix_mautrix_facebook_docker_image }} \

From a3406a182bf9590d7ee018867ee4fbd02c6473b5 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:04:47 +0200
Subject: [PATCH 26/79] Move some things around

---
 .../defaults/main.yml                           |  2 +-
 .../tasks/setup_install.yml                     | 17 ++++++++---------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index 95adf354004..d0b3968209d 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -33,7 +33,6 @@ matrix_mautrix_facebook_appservice_token: ''
 matrix_mautrix_facebook_homeserver_token: ''
 
 
-
 # Database-related configuration fields.
 #
 # To use SQLite, stick to these defaults.
@@ -61,6 +60,7 @@ matrix_mautrix_facebook_appservice_database: "{{
 	}[matrix_mautrix_facebook_database_engine]
 }}"
 
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_facebook_login_shared_secret: ''
 
diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
index ef814108862..c3c8d61fee6 100644
--- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
@@ -1,5 +1,13 @@
 ---
 
+# If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist.
+# We don't want to fail in such cases.
+- name: Fail if matrix-synapse role already executed
+  fail:
+    msg: >-
+      The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role.
+  when: "matrix_synapse_role_executed|default(False)"
+
 - block:
   - name: Check if an SQLite database already exists
     stat:
@@ -20,15 +28,6 @@
     when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mautrix_facebook_database_engine == 'postgres'"
 
-
-# If the matrix-synapse role is not used, `matrix_synapse_role_executed` won't exist.
-# We don't want to fail in such cases.
-- name: Fail if matrix-synapse role already executed
-  fail:
-    msg: >-
-      The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role.
-  when: "matrix_synapse_role_executed|default(False)"
-
 - name: Ensure Mautrix Facebook image is pulled
   docker_image:
     name: "{{ matrix_mautrix_facebook_docker_image }}"

From 9bf8ce878e9451f2f3aa24d200d9d10dbf08e8b3 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:06:23 +0200
Subject: [PATCH 27/79] Add (Postgres + SQLite) support to
 matrix-mautrix-telegram bridge

I don't use this bridge, so this is completely untested.
---
 .../defaults/main.yml                         | 29 +++++++++++++++++++
 .../tasks/setup_install.yml                   | 20 +++++++++++++
 .../templates/config.yaml.j2                  |  2 +-
 3 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index b6f4ef5bb35..d0df6a7d303 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -43,6 +43,35 @@ matrix_mautrix_telegram_systemd_wanted_services_list: []
 matrix_mautrix_telegram_appservice_token: ''
 matrix_mautrix_telegram_homeserver_token: ''
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_telegram_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_telegram_postgres_*` variables
+matrix_mautrix_telegram_database_engine: 'sqlite'
+
+matrix_mautrix_telegram_sqlite_database_path_local: "{{ matrix_mautrix_telegram_data_path }}/mautrix-telegram.db"
+matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegram.db"
+
+matrix_mautrix_telegram_postgres_username: 'matrix_mautrix_telegram'
+matrix_mautrix_telegram_postgres_password: 'some-password'
+matrix_mautrix_telegram_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_telegram_postgres_port: 5432
+matrix_mautrix_telegram_postgres_db_name: 'matrix_mautrix_telegram'
+
+matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_postgres_username }}:{{ matrix_mautrix_telegram_postgres_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_postgres_port }}/{{ matrix_mautrix_telegram_postgres_db_name }}'
+
+matrix_mautrix_telegram_appservice_database: "{{
+	{
+		'sqlite': matrix_mautrix_telegram_sqlite_database_path_in_container,
+		'postgres': matrix_mautrix_telegram_postgres_connection_string,
+	}[matrix_mautrix_telegram_database_engine]
+}}"
+
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_telegram_login_shared_secret: ''
 
diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
index 9dc38ec2c3f..db1a8f832c8 100644
--- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
@@ -8,6 +8,26 @@
       The matrix-bridge-mautrix-telegram role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}"
+    register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_telegram_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_telegram_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_telegram_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_telegram_database_engine == 'postgres'"
+
 - name: Ensure Mautrix Telegram image is pulled
   docker_image:
     name: "{{ matrix_mautrix_telegram_docker_image }}"
diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
index 65f17d338a2..52efba02d73 100644
--- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: postgres://mautrix_bridge_telegram:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_bridge_telegram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_telegram
+    database: {{ matrix_mautrix_telegram_appservice_database|to_json }}
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in

From 05dd091133ad7833ca544bd4ccda55311c0450c5 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:11:41 +0200
Subject: [PATCH 28/79] Rename `_database_hostname` to `_postgres_hostname`

Should have been done during 2a99e84b5bfd9f39, but I've missed it.

Because of this, 9bf8ce878e945 was also incomplete and would have caused
an error.
---
 group_vars/matrix_servers | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 5e3695cac51..0bc0c0c5431 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -958,12 +958,12 @@ matrix_postgres_additional_databases: |
       'name': matrix_appservice_slack_postgres_db_name,
       'username': matrix_appservice_slack_postgres_username,
       'password': matrix_appservice_slack_postgres_password,
-    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_appservice_irc_postgres_db_name,
       'username': matrix_appservice_irc_postgres_username,
       'password': matrix_appservice_irc_postgres_password,
-    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_facebook_postgres_db_name,
       'username': matrix_mautrix_facebook_postgres_username,
@@ -973,17 +973,17 @@ matrix_postgres_additional_databases: |
       'name': matrix_mautrix_hangouts_postgres_db_name,
       'username': matrix_mautrix_hangouts_postgres_username,
       'password': matrix_mautrix_hangouts_postgres_password,
-    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_telegram_postgres_db_name,
       'username': matrix_mautrix_telegram_postgres_username,
       'password': matrix_mautrix_telegram_postgres_password,
-    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_whatsapp_postgres_db_name,
       'username': matrix_mautrix_whatsapp_postgres_username,
       'password': matrix_mautrix_whatsapp_postgres_password,
-    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
@@ -993,45 +993,45 @@ matrix_postgres_additional_databases: |
       'name': matrix_mx_puppet_skype_postgres_db_name,
       'username': matrix_mx_puppet_skype_postgres_username,
       'password': matrix_mx_puppet_skype_postgres_password,
-    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mx_puppet_slack_postgres_db_name,
       'username': matrix_mx_puppet_slack_postgres_username,
       'password': matrix_mx_puppet_slack_postgres_password,
-    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_postgres_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_twitter_postgres_db_name,
       'username': matrix_mx_puppet_twitter_postgres_username,
       'password': matrix_mx_puppet_twitter_postgres_password,
-    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_postgres_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_instagram_postgres_db_name,
       'username': matrix_mx_puppet_instagram_postgres_username,
       'password': matrix_mx_puppet_instagram_postgres_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
-           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_instagram_postgres_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_discord_postgres_db_name,
       'username': matrix_mx_puppet_discord_postgres_username,
       'password': matrix_mx_puppet_discord_postgres_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
-           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_discord_postgres_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_steam_postgres_db_name,
       'username': matrix_mx_puppet_steam_postgres_username,
       'password': matrix_mx_puppet_steam_postgres_password,
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
-           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_steam_postgres_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_dimension_postgres_db_name,
       'username': matrix_dimension_postgres_username,
       'password': matrix_dimension_postgres_password,
     }] if (matrix_dimension_enabled
            and matrix_dimension_database_engine == 'postgres'
-           and matrix_dimension_database_hostname == 'matrix-postgres') else [])
+           and matrix_dimension_postgres_hostname == 'matrix-postgres') else [])
    }}
 
 ######################################################################

From 6a2dd8088cea10713ffd760d32cc789821ae5ae9 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:13:10 +0200
Subject: [PATCH 29/79] Fix typo (matrix_telegram_facebook_postgres_hostname ->
 matrix_mautrix_telegram_postgres_hostname)

Related to 9bf8ce878e945.
---
 group_vars/matrix_servers | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 0bc0c0c5431..626498c0036 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -978,7 +978,7 @@ matrix_postgres_additional_databases: |
       'name': matrix_mautrix_telegram_postgres_db_name,
       'username': matrix_mautrix_telegram_postgres_username,
       'password': matrix_mautrix_telegram_postgres_password,
-    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_telegram_facebook_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_whatsapp_postgres_db_name,
       'username': matrix_mautrix_whatsapp_postgres_username,

From 2848322461e451d2fb1985ecf797d87e5765ea03 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:19:35 +0200
Subject: [PATCH 30/79] Remove some remaining `_database_password` usage in
 favor of `_postgres_pasword`

The only one that remains is `matrix_synapse_database_password`, but
that's something old and should be dealt with separately in the future
(unless it remains as it is).
---
 group_vars/matrix_servers | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 626498c0036..e3f16d3f718 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -142,7 +142,7 @@ matrix_appservice_slack_systemd_required_services_list: |
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
+matrix_appservice_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -181,7 +181,7 @@ matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key |
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
+matrix_appservice_irc_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
 
 
 ######################################################################
@@ -219,7 +219,7 @@ matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
+matrix_mautrix_facebook_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -256,7 +256,7 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
+matrix_mautrix_hangouts_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -296,7 +296,7 @@ matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
+matrix_mautrix_telegram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -328,7 +328,7 @@ matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
+matrix_mautrix_whatsapp_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -388,7 +388,7 @@ matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
+matrix_mx_puppet_skype_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -423,7 +423,7 @@ matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
+matrix_mx_puppet_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -459,7 +459,7 @@ matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_p
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
+matrix_mx_puppet_twitter_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -494,7 +494,7 @@ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
+matrix_mx_puppet_instagram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -528,7 +528,7 @@ matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_pr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
+matrix_mx_puppet_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -562,7 +562,7 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
+matrix_mx_puppet_steam_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -676,7 +676,7 @@ matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if m
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
+matrix_dimension_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
 
 ######################################################################
 #

From 6c77eae96935f6a78c6ad170134d0797b18520af Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:24:37 +0200
Subject: [PATCH 31/79] Add (Postgres + SQLite) support to
 matrix-mautrix-whatsapp bridge

I don't use this bridge, so this is completely untested.
---
 .../defaults/main.yml                         | 36 +++++++++++++++++++
 .../tasks/setup_install.yml                   | 26 ++++++++++++--
 .../templates/config.yaml.j2                  |  4 +--
 3 files changed, 61 insertions(+), 5 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
index e9929f96ead..6cabea3ec64 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
@@ -27,6 +27,42 @@ matrix_mautrix_whatsapp_systemd_wanted_services_list: []
 matrix_mautrix_whatsapp_appservice_token: ''
 matrix_mautrix_whatsapp_homeserver_token: ''
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_whatsapp_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_whatsapp_postgres_*` variables
+matrix_mautrix_whatsapp_database_engine: 'sqlite'
+
+matrix_mautrix_whatsapp_sqlite_database_path_local: "{{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db"
+matrix_mautrix_whatsapp_sqlite_database_path_in_container: "/data/mautrix-whatsapp.db"
+
+matrix_mautrix_whatsapp_postgres_username: 'matrix_mautrix_whatsapp'
+matrix_mautrix_whatsapp_postgres_password: 'some-password'
+matrix_mautrix_whatsapp_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_whatsapp_postgres_port: 5432
+matrix_mautrix_whatsapp_postgres_db_name: 'matrix_mautrix_whatsapp'
+
+matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_postgres_username }}:{{ matrix_mautrix_whatsapp_postgres_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_postgres_port }}/{{ matrix_mautrix_whatsapp_postgres_db_name }}'
+
+matrix_mautrix_whatsapp_appservice_database_type: "{{
+	{
+		'sqlite': 'sqlite3',
+		'postgres':'postgres',
+	}[matrix_mautrix_whatsapp_database_engine]
+}}"
+
+matrix_mautrix_whatsapp_appservice_database_uri: "{{
+	{
+		'sqlite': matrix_mautrix_whatsapp_sqlite_database_path_in_container,
+		'postgres': matrix_mautrix_whatsapp_postgres_connection_string,
+	}[matrix_mautrix_whatsapp_database_engine]
+}}"
+
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_whatsapp_login_shared_secret: ''
 
diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
index 6bd87bbe2c1..15453e8caf0 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
@@ -8,6 +8,26 @@
       The matrix-bridge-mautrix-whatsapp role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}"
+    register: matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_whatsapp_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_whatsapp_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_whatsapp_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_whatsapp_database_engine == 'postgres'"
+
 - name: Ensure Mautrix Whatsapp image is pulled
   docker_image:
     name: "{{ matrix_mautrix_whatsapp_docker_image }}"
@@ -26,12 +46,12 @@
     - "{{ matrix_mautrix_whatsapp_base_path }}"
     - "{{ matrix_mautrix_whatsapp_config_path }}"
     - "{{ matrix_mautrix_whatsapp_data_path }}"
-    
+
 - name: Check if an old database file exists
   stat:
     path: "{{ matrix_mautrix_whatsapp_base_path }}/mautrix-whatsapp.db"
   register: matrix_mautrix_whatsapp_stat_database
-  
+
 - name: Check if an old matrix state file exists
   stat:
     path: "{{ matrix_mautrix_whatsapp_base_path }}/mx-state.json"
@@ -48,7 +68,7 @@
 - name: (Data relocation) Move mautrix-whatsapp database file to ./data directory
   command: "mv {{ matrix_mautrix_whatsapp_base_path }}/mautrix-whatsapp.db {{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db"
   when: "matrix_mautrix_whatsapp_stat_database.stat.exists"
-  
+
 - name: (Data relocation) Move mautrix-whatsapp mx-state file to ./data directory
   command: "mv {{ matrix_mautrix_whatsapp_base_path }}/mx-state.json {{ matrix_mautrix_whatsapp_data_path }}/mx-state.json"
   when: "matrix_mautrix_whatsapp_stat_mx_state.stat.exists"
diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
index 93956049ced..8921669567a 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2
@@ -19,11 +19,11 @@ appservice:
     # Database config.
     database:
         # The database type. "sqlite3" and "postgres" are supported.
-        type: sqlite3
+        type: {{ matrix_mautrix_whatsapp_appservice_database_type|to_json }}
         # The database URI.
         #   SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string
         #   Postgres: Connection string. For example, postgres://user:password@host/database
-        uri: postgres://matrix_bridge_whatsapp@{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_bridge_whatsapp') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_bridge_whatsapp
+        uri: {{ matrix_mautrix_whatsapp_appservice_database_uri|to_json }}
         # Maximum number of connections. Mostly relevant for Postgres.
         max_open_conns: 20
         max_idle_conns: 2

From 43d6ff2af83abf0610c73b507eb4d7acae045052 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:30:10 +0200
Subject: [PATCH 32/79] Fix sqlite usage for mautrix-facebook/mautrix-telegram

Regression since 2d99ade72f and 9bf8ce878e94, respectively.

When SQLite is to be used, these bridges expect an `sqlite://`
connection string, and not a plain file name (path), like Appservice
Discord and mautrix-whatsapp do.
---
 roles/matrix-bridge-mautrix-facebook/defaults/main.yml | 2 +-
 roles/matrix-bridge-mautrix-telegram/defaults/main.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index d0b3968209d..97584997872 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -55,7 +55,7 @@ matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_maut
 
 matrix_mautrix_facebook_appservice_database: "{{
 	{
-		'sqlite': matrix_mautrix_facebook_sqlite_database_path_in_container,
+		'sqlite': ('sqlite://' + matrix_mautrix_facebook_sqlite_database_path_in_container),
 		'postgres': matrix_mautrix_facebook_postgres_connection_string,
 	}[matrix_mautrix_facebook_database_engine]
 }}"
diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index d0df6a7d303..9ba9be095f8 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -66,7 +66,7 @@ matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_maut
 
 matrix_mautrix_telegram_appservice_database: "{{
 	{
-		'sqlite': matrix_mautrix_telegram_sqlite_database_path_in_container,
+		'sqlite': ('sqlite://' + matrix_mautrix_telegram_sqlite_database_path_in_container),
 		'postgres': matrix_mautrix_telegram_postgres_connection_string,
 	}[matrix_mautrix_telegram_database_engine]
 }}"

From ce21ea3640283889be6ed7c526116ca6c47ca921 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 12:34:59 +0200
Subject: [PATCH 33/79] Add (Postgres + SQLite) support to
 matrix-mautrix-hangouts bridge

I don't use this bridge, so this is completely untested.
---
 .../defaults/main.yml                         | 29 +++++++++++++++++++
 .../tasks/setup_install.yml                   | 20 +++++++++++++
 .../templates/config.yaml.j2                  |  2 +-
 .../matrix-mautrix-hangouts.service.j2        |  1 +
 4 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
index ab670052853..af0fe061abd 100644
--- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
@@ -39,6 +39,35 @@ matrix_mautrix_hangouts_systemd_wanted_services_list: []
 matrix_mautrix_hangouts_appservice_token: ''
 matrix_mautrix_hangouts_homeserver_token: ''
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_mautrix_hangouts_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_mautrix_hangouts_postgres_*` variables
+matrix_mautrix_hangouts_database_engine: 'sqlite'
+
+matrix_mautrix_hangouts_sqlite_database_path_local: "{{ matrix_mautrix_hangouts_data_path }}/mautrix-hangouts.db"
+matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangouts.db"
+
+matrix_mautrix_hangouts_postgres_username: 'matrix_mautrix_hangouts'
+matrix_mautrix_hangouts_postgres_password: 'some-password'
+matrix_mautrix_hangouts_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_hangouts_postgres_port: 5432
+matrix_mautrix_hangouts_postgres_db_name: 'matrix_mautrix_hangouts'
+
+matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_postgres_username }}:{{ matrix_mautrix_hangouts_postgres_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_postgres_port }}/{{ matrix_mautrix_hangouts_postgres_db_name }}'
+
+matrix_mautrix_hangouts_appservice_database: "{{
+	{
+		'sqlite': ('sqlite://' + matrix_mautrix_hangouts_sqlite_database_path_in_container),
+		'postgres': matrix_mautrix_hangouts_postgres_connection_string,
+	}[matrix_mautrix_hangouts_database_engine]
+}}"
+
+
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mautrix_hangouts_login_shared_secret: ''
 
diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
index eec5e00675a..9e7bfbba9ba 100644
--- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
@@ -8,6 +8,26 @@
       The matrix-bridge-mautrix-hangouts role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}"
+    register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mautrix_hangouts_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mautrix_hangouts_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_hangouts_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_postgres_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mautrix_hangouts_database_engine == 'postgres'"
+
 - name: Ensure Mautrix Hangouts image is pulled
   docker_image:
     name: "{{ matrix_mautrix_hangouts_docker_image }}"
diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
index e4e59ad26d9..cc2ca90bb99 100644
--- a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2
@@ -27,7 +27,7 @@ appservice:
     # Format examples:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
-    database: postgres://mautrix_bridge_hangouts:{{ matrix_additional_databases | selectattr('name', 'equalto', 'mautrix_bridge_hangouts') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/mautrix_bridge_hangouts
+    database: {{ matrix_mautrix_hangouts_appservice_database|to_json }}
 
     # The unique ID of this appservice.
     id: hangouts
diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2
index 26280da32af..395591905f5 100644
--- a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2
+++ b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2
@@ -18,6 +18,7 @@ ExecStartPre={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-han
 			--log-driver=none \
 			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
 			--cap-drop=ALL \
+			--network={{ matrix_docker_network }} \
 			-v {{ matrix_mautrix_hangouts_config_path }}:/config:z \
 			-v {{ matrix_mautrix_hangouts_data_path }}:/data:z \
 			{{ matrix_mautrix_hangouts_docker_image }} \

From 087dbe4ddc80ba6308e7ee98391ea475354e8860 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 13:02:47 +0200
Subject: [PATCH 34/79] Rename `matrix_*_postgres_*` back to
 `matrix_*_database_*`

I was thinking that it makes sense to be more specific,
and using `_postgres_` also separated these variables
from the `_database_` variables that ended up in bridge configuration.

However, @jdreichmann makes a good point
(https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740#discussion_r542281102)
that we don't need to be so specific and can allow for other engines (like MySQL) to use these variables.
---
 group_vars/matrix_servers                     | 112 +++++++++---------
 .../defaults/main.yml                         |  10 +-
 .../defaults/main.yml                         |  10 +-
 .../defaults/main.yml                         |  10 +-
 .../defaults/main.yml                         |  10 +-
 .../defaults/main.yml                         |  10 +-
 6 files changed, 81 insertions(+), 81 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index e3f16d3f718..acf133e128d 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -70,7 +70,7 @@ matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_k
 # We only make this use Postgres if our own Postgres server is enabled.
 # It's only then (for now) that we can automatically create the necessary database and user for this service.
 matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}"
+matrix_appservice_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -142,7 +142,7 @@ matrix_appservice_slack_systemd_required_services_list: |
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
+matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -181,7 +181,7 @@ matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key |
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_appservice_irc_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
+matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
 
 
 ######################################################################
@@ -219,7 +219,7 @@ matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_facebook_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
+matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -256,7 +256,7 @@ matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_hangouts_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
+matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -296,7 +296,7 @@ matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_telegram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
+matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -328,7 +328,7 @@ matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_pro
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mautrix_whatsapp_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
+matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -388,7 +388,7 @@ matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_skype_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
+matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -423,7 +423,7 @@ matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_slack_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
+matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -459,7 +459,7 @@ matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_p
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_twitter_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
+matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -494,7 +494,7 @@ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_instagram_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
+matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -528,7 +528,7 @@ matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_pr
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_discord_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
+matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -562,7 +562,7 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_mx_puppet_steam_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
+matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -676,7 +676,7 @@ matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if m
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_dimension_postgres_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
+matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -950,39 +950,39 @@ matrix_postgres_db_name: "homeserver"
 matrix_postgres_additional_databases: |
   {{
     ([{
-      'name': matrix_appservice_discord_postgres_db_name,
-      'username': matrix_appservice_discord_postgres_username,
-      'password': matrix_appservice_discord_postgres_password,
+      'name': matrix_appservice_discord_database_db_name,
+      'username': matrix_appservice_discord_database_username,
+      'password': matrix_appservice_discord_database_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_slack_postgres_db_name,
-      'username': matrix_appservice_slack_postgres_username,
-      'password': matrix_appservice_slack_postgres_password,
+      'name': matrix_appservice_slack_database_db_name,
+      'username': matrix_appservice_slack_database_username,
+      'password': matrix_appservice_slack_database_password,
     }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_irc_postgres_db_name,
-      'username': matrix_appservice_irc_postgres_username,
-      'password': matrix_appservice_irc_postgres_password,
+      'name': matrix_appservice_irc_database_db_name,
+      'username': matrix_appservice_irc_database_username,
+      'password': matrix_appservice_irc_database_password,
     }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_facebook_postgres_db_name,
-      'username': matrix_mautrix_facebook_postgres_username,
-      'password': matrix_mautrix_facebook_postgres_password,
+      'name': matrix_mautrix_facebook_database_db_name,
+      'username': matrix_mautrix_facebook_database_username,
+      'password': matrix_mautrix_facebook_database_password,
     }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_hangouts_postgres_db_name,
-      'username': matrix_mautrix_hangouts_postgres_username,
-      'password': matrix_mautrix_hangouts_postgres_password,
+      'name': matrix_mautrix_hangouts_database_db_name,
+      'username': matrix_mautrix_hangouts_database_username,
+      'password': matrix_mautrix_hangouts_database_password,
     }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_telegram_postgres_db_name,
-      'username': matrix_mautrix_telegram_postgres_username,
-      'password': matrix_mautrix_telegram_postgres_password,
+      'name': matrix_mautrix_telegram_database_db_name,
+      'username': matrix_mautrix_telegram_database_username,
+      'password': matrix_mautrix_telegram_database_password,
     }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_whatsapp_postgres_db_name,
-      'username': matrix_mautrix_whatsapp_postgres_username,
-      'password': matrix_mautrix_whatsapp_postgres_password,
+      'name': matrix_mautrix_whatsapp_database_db_name,
+      'username': matrix_mautrix_whatsapp_database_username,
+      'password': matrix_mautrix_whatsapp_database_password,
     }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_postgres_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_bridge_sms',
@@ -990,45 +990,45 @@ matrix_postgres_additional_databases: |
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      'name': matrix_mx_puppet_skype_postgres_db_name,
-      'username': matrix_mx_puppet_skype_postgres_username,
-      'password': matrix_mx_puppet_skype_postgres_password,
+      'name': matrix_mx_puppet_skype_database_db_name,
+      'username': matrix_mx_puppet_skype_database_username,
+      'password': matrix_mx_puppet_skype_database_password,
     }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_postgres_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mx_puppet_slack_postgres_db_name,
-      'username': matrix_mx_puppet_slack_postgres_username,
-      'password': matrix_mx_puppet_slack_postgres_password,
+      'name': matrix_mx_puppet_slack_database_db_name,
+      'username': matrix_mx_puppet_slack_database_username,
+      'password': matrix_mx_puppet_slack_database_password,
     }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_postgres_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_twitter_postgres_db_name,
-      'username': matrix_mx_puppet_twitter_postgres_username,
-      'password': matrix_mx_puppet_twitter_postgres_password,
+      'name': matrix_mx_puppet_twitter_database_db_name,
+      'username': matrix_mx_puppet_twitter_database_username,
+      'password': matrix_mx_puppet_twitter_database_password,
     }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_postgres_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_instagram_postgres_db_name,
-      'username': matrix_mx_puppet_instagram_postgres_username,
-      'password': matrix_mx_puppet_instagram_postgres_password,
+      'name': matrix_mx_puppet_instagram_database_db_name,
+      'username': matrix_mx_puppet_instagram_database_username,
+      'password': matrix_mx_puppet_instagram_database_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
            and matrix_mx_puppet_instagram_postgres_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_discord_postgres_db_name,
-      'username': matrix_mx_puppet_discord_postgres_username,
-      'password': matrix_mx_puppet_discord_postgres_password,
+      'name': matrix_mx_puppet_discord_database_db_name,
+      'username': matrix_mx_puppet_discord_database_username,
+      'password': matrix_mx_puppet_discord_database_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
            and matrix_mx_puppet_discord_postgres_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_steam_postgres_db_name,
-      'username': matrix_mx_puppet_steam_postgres_username,
-      'password': matrix_mx_puppet_steam_postgres_password,
+      'name': matrix_mx_puppet_steam_database_db_name,
+      'username': matrix_mx_puppet_steam_database_username,
+      'password': matrix_mx_puppet_steam_database_password,
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
            and matrix_mx_puppet_steam_postgres_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_dimension_postgres_db_name,
-      'username': matrix_dimension_postgres_username,
-      'password': matrix_dimension_postgres_password,
+      'name': matrix_dimension_database_db_name,
+      'username': matrix_dimension_database_username,
+      'password': matrix_dimension_database_password,
     }] if (matrix_dimension_enabled
            and matrix_dimension_database_engine == 'postgres'
            and matrix_dimension_postgres_hostname == 'matrix-postgres') else [])
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index fb517cd5f6e..57a888671e4 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -53,16 +53,16 @@ matrix_appservice_discord_database_engine: 'sqlite'
 matrix_appservice_discord_sqlite_database_path_local: "{{ matrix_appservice_discord_data_path }}/discord.db"
 matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db"
 
-matrix_appservice_discord_postgres_username: 'matrix_appservice_discord'
-matrix_appservice_discord_postgres_password: 'some-password'
+matrix_appservice_discord_database_username: 'matrix_appservice_discord'
+matrix_appservice_discord_database_password: 'some-password'
 matrix_appservice_discord_postgres_hostname: 'matrix-postgres'
-matrix_appservice_discord_postgres_port: 5432
-matrix_appservice_discord_postgres_db_name: 'matrix_appservice_discord'
+matrix_appservice_discord_database_port: 5432
+matrix_appservice_discord_database_db_name: 'matrix_appservice_discord'
 
 # These 2 variables are what actually ends up in the bridge configuration.
 # It's best if you don't change them directly, but rather redefine the sub-variables that constitute them.
 matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}"
-matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_postgres_username }}:{{ matrix_appservice_discord_postgres_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_postgres_port }}/{{ matrix_appservice_discord_postgres_db_name }}'
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}'
 
 
 # Tells whether the bot should make use of "Privileged Gateway Intents".
diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index 97584997872..246c32fe683 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -45,13 +45,13 @@ matrix_mautrix_facebook_database_engine: 'sqlite'
 matrix_mautrix_facebook_sqlite_database_path_local: "{{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db"
 matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebook.db"
 
-matrix_mautrix_facebook_postgres_username: 'matrix_mautrix_facebook'
-matrix_mautrix_facebook_postgres_password: 'some-password'
+matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook'
+matrix_mautrix_facebook_database_password: 'some-password'
 matrix_mautrix_facebook_postgres_hostname: 'matrix-postgres'
-matrix_mautrix_facebook_postgres_port: 5432
-matrix_mautrix_facebook_postgres_db_name: 'matrix_mautrix_facebook'
+matrix_mautrix_facebook_database_port: 5432
+matrix_mautrix_facebook_database_db_name: 'matrix_mautrix_facebook'
 
-matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_postgres_username }}:{{ matrix_mautrix_facebook_postgres_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_postgres_port }}/{{ matrix_mautrix_facebook_postgres_db_name }}'
+matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}'
 
 matrix_mautrix_facebook_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
index af0fe061abd..04280ac42a6 100644
--- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
@@ -52,13 +52,13 @@ matrix_mautrix_hangouts_database_engine: 'sqlite'
 matrix_mautrix_hangouts_sqlite_database_path_local: "{{ matrix_mautrix_hangouts_data_path }}/mautrix-hangouts.db"
 matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangouts.db"
 
-matrix_mautrix_hangouts_postgres_username: 'matrix_mautrix_hangouts'
-matrix_mautrix_hangouts_postgres_password: 'some-password'
+matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts'
+matrix_mautrix_hangouts_database_password: 'some-password'
 matrix_mautrix_hangouts_postgres_hostname: 'matrix-postgres'
-matrix_mautrix_hangouts_postgres_port: 5432
-matrix_mautrix_hangouts_postgres_db_name: 'matrix_mautrix_hangouts'
+matrix_mautrix_hangouts_database_port: 5432
+matrix_mautrix_hangouts_database_db_name: 'matrix_mautrix_hangouts'
 
-matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_postgres_username }}:{{ matrix_mautrix_hangouts_postgres_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_postgres_port }}/{{ matrix_mautrix_hangouts_postgres_db_name }}'
+matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}'
 
 matrix_mautrix_hangouts_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index 9ba9be095f8..4f584128e4e 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -56,13 +56,13 @@ matrix_mautrix_telegram_database_engine: 'sqlite'
 matrix_mautrix_telegram_sqlite_database_path_local: "{{ matrix_mautrix_telegram_data_path }}/mautrix-telegram.db"
 matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegram.db"
 
-matrix_mautrix_telegram_postgres_username: 'matrix_mautrix_telegram'
-matrix_mautrix_telegram_postgres_password: 'some-password'
+matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram'
+matrix_mautrix_telegram_database_password: 'some-password'
 matrix_mautrix_telegram_postgres_hostname: 'matrix-postgres'
-matrix_mautrix_telegram_postgres_port: 5432
-matrix_mautrix_telegram_postgres_db_name: 'matrix_mautrix_telegram'
+matrix_mautrix_telegram_database_port: 5432
+matrix_mautrix_telegram_database_db_name: 'matrix_mautrix_telegram'
 
-matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_postgres_username }}:{{ matrix_mautrix_telegram_postgres_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_postgres_port }}/{{ matrix_mautrix_telegram_postgres_db_name }}'
+matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}'
 
 matrix_mautrix_telegram_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
index 6cabea3ec64..357f5892599 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
@@ -40,13 +40,13 @@ matrix_mautrix_whatsapp_database_engine: 'sqlite'
 matrix_mautrix_whatsapp_sqlite_database_path_local: "{{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db"
 matrix_mautrix_whatsapp_sqlite_database_path_in_container: "/data/mautrix-whatsapp.db"
 
-matrix_mautrix_whatsapp_postgres_username: 'matrix_mautrix_whatsapp'
-matrix_mautrix_whatsapp_postgres_password: 'some-password'
+matrix_mautrix_whatsapp_database_username: 'matrix_mautrix_whatsapp'
+matrix_mautrix_whatsapp_database_password: 'some-password'
 matrix_mautrix_whatsapp_postgres_hostname: 'matrix-postgres'
-matrix_mautrix_whatsapp_postgres_port: 5432
-matrix_mautrix_whatsapp_postgres_db_name: 'matrix_mautrix_whatsapp'
+matrix_mautrix_whatsapp_database_port: 5432
+matrix_mautrix_whatsapp_database_db_name: 'matrix_mautrix_whatsapp'
 
-matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_postgres_username }}:{{ matrix_mautrix_whatsapp_postgres_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_postgres_port }}/{{ matrix_mautrix_whatsapp_postgres_db_name }}'
+matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}'
 
 matrix_mautrix_whatsapp_appservice_database_type: "{{
 	{

From 5d70bc1376f30c4453beffe0c23ccb4c3004635c Mon Sep 17 00:00:00 2001
From: transcaffeine <transcaffeine@finallycoffee.eu>
Date: Mon, 14 Dec 2020 12:38:51 +0100
Subject: [PATCH 35/79] add postgres support for mx-puppet-* with fallback to
 sqlite in role and migration notice

---
 .../defaults/main.yml                         | 15 +++++++++++++
 .../tasks/validate_config.yml                 | 20 +++++++++++++++++
 .../templates/config.yaml.j2                  | 11 ++++++----
 .../defaults/main.yml                         | 14 ++++++++++++
 .../tasks/validate_config.yml                 | 21 ++++++++++++++++++
 .../templates/config.yaml.j2                  |  7 ++++--
 .../defaults/main.yml                         | 15 +++++++++++++
 .../tasks/validate_config.yml                 | 20 +++++++++++++++++
 .../templates/config.yaml.j2                  |  7 ++++--
 .../defaults/main.yml                         | 15 +++++++++++++
 .../tasks/validate_config.yml                 | 20 +++++++++++++++++
 .../templates/config.yaml.j2                  |  7 ++++--
 .../defaults/main.yml                         | 14 ++++++++++++
 .../tasks/validate_config.yml                 | 22 +++++++++++++++++++
 .../templates/config.yaml.j2                  |  7 ++++--
 .../defaults/main.yml                         | 15 +++++++++++++
 .../tasks/validate_config.yml                 | 22 +++++++++++++++++++
 .../templates/config.yaml.j2                  |  7 ++++--
 18 files changed, 245 insertions(+), 14 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
index 268bbf8f8c0..5a65f33e944 100644
--- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
@@ -55,6 +55,21 @@ matrix_mx_puppet_discord_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_discord_login_shared_secret: ''
 
+# Database configuration
+matrix_mx_puppet_discord_database_engine: 'sqlite'
+matrix_mx_puppet_discord_database_username: matrix_mx_puppet_discord
+matrix_mx_puppet_discord_database_password: ~
+matrix_mx_puppet_discord_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_discord_database_port: 5432
+matrix_mx_puppet_discord_database_name: matrix_mx_puppet_discord
+matrix_mx_puppet_discord_database_file: /data/database.db
+matrix_mx_puppet_discord_database_connString: >-2
+  {%- if matrix_mx_puppet_discord_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_discord_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_discord_database_engine }}://{{ matrix_mx_puppet_discord_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
index c253eda296e..8db47ed1f92 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
@@ -8,3 +8,23 @@
   with_items:
     - "matrix_mx_puppet_discord_appservice_token"
     - "matrix_mx_puppet_discord_homeserver_token"
+
+- block:
+  - name: Check if a SQLite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_discord_data_path }}/database.db"
+    register: matrix_mx_puppet_discord_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-2
+        matrix_mx_puppet_discord_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_discord_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_discord_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_discord_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_discord_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_discord_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
index 88c591e2f63..c24c7e54cda 100644
--- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
@@ -100,15 +100,18 @@ namePatterns:
   group: :name
 
 database:
+{% if matrix_mx_puppet_slack_discord_database_engine == 'sqlite' %}
+  # Use SQLite3 as a database backend
+  # The name of the database file
+  filename: /data/database.db
+{% else %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_discord:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_discord') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_discord?sslmode=disable"
-  # Use SQLite3 as a database backend
-  # The name of the database file
-  #filename: /data/database.db
+  connString: {{ matrix_mx_puppet_discord_database_connString|to_json }}
+{% endif %}
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
index 21cde53fa5b..c30f9b8b386 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
@@ -46,6 +46,20 @@ matrix_mx_puppet_instagram_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_instagram_login_shared_secret: ''
 
+matrix_mx_puppet_instagram_database_engine: sqlite
+matrix_mx_puppet_instagram_database_username: matrix_mx_puppet_instagram
+matrix_mx_puppet_instagram_database_password: ~
+matrix_mx_puppet_instagram_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_instagram_database_port: 5432
+matrix_mx_puppet_instagram_database_name: matrix_mx_puppet_instagram
+matrix_mx_puppet_instagram_database_file: /data/database.db
+matrix_mx_puppet_instagram_database_connString: >-2
+  {%- if matrix_mx_puppet_instagram_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_instagram_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_instagram_database_engine }}://{{ matrix_mx_puppet_instagram_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
index b6d9d994e3a..7000a8a175b 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
@@ -8,3 +8,24 @@
   with_items:
     - "matrix_mx_puppet_instagram_appservice_token"
     - "matrix_mx_puppet_instagram_homeserver_token"
+
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_instagram_data_path }}/database.db"
+    register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result
+
+   - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mx_puppet_instagram_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_instagram_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_instagram_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_instagram_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_instagram_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_instagram_database_engine == 'postgres'"
+
diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
index 2d1891957db..d89c5bdb23d 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
@@ -44,15 +44,18 @@ provisioning:
   apiPrefix: /_matrix/provision
 
 database:
+{% if matrix_mx_puppet_instagram_database_engine == 'postgres' %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_instagram:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_instagram') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_instagram?sslmode=disable"
+  connString: {{ matrix_mx_puppet_instagram_database_connString | to_json }}
+{% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  #filename: /data/database.db
+  filename: {{ matrix_mx_puppet_instagram_database_file }}
+{% endif %}
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
index 0c16a8babea..3b7448c66b5 100644
--- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
@@ -53,6 +53,21 @@ matrix_mx_puppet_skype_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_skype_login_shared_secret: ''
 
+# Database configuration, role default is `sqlite` but playbook default is `postgres`
+matrix_mx_puppet_skype_database_engine: sqlite
+matrix_mx_puppet_skype_database_username: matrix_mx_puppet_skype
+matrix_mx_puppet_skype_database_password: ~
+matrix_mx_puppet_skype_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_skype_database_port: 5432
+matrix_mx_puppet_skype_database_name: matrix_mx_puppet_skype
+matrix_mx_puppet_skype_database_file: /data/database.db
+matrix_mx_puppet_skype_database_connString: >-2
+  {%- if matrix_mx_puppet_skype_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_skype_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_skype_database_engine }}://{{ matrix_mx_puppet_skype_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
index 7ed433b12f7..c7100e51ab2 100644
--- a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
@@ -8,3 +8,23 @@
   with_items:
     - "matrix_mx_puppet_skype_appservice_token"
     - "matrix_mx_puppet_skype_homeserver_token"
+
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_skype_data_path }}/database.db"
+    register: matrix_mx_puppet_skype_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mx_puppet_skype_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_skype_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_skype_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_skype_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_skype_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_skype_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_skype_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
index a54ca758b34..a32523b1cda 100644
--- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
@@ -68,15 +68,18 @@ logging:
       #  - Store
 
 database:
+{% if matrix_mx_puppet_skype_database_engine == 'postgres' %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_skype:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_skype') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_skype?sslmode=disable"
+  connString: {{ matrix_mx_puppet_skype_database_connString | to_json }}
+{% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  #filename: /data/database.db
+  filename: {{ matrix_mx_puppet_skype_database_file }}
+{% endif %}
 
 provisioning:
   # Regex of Matrix IDs allowed to use the puppet bridge
diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
index 96ab7833160..5f91beb8db6 100644
--- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
@@ -57,6 +57,21 @@ matrix_mx_puppet_slack_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_slack_login_shared_secret: ''
 
+# Database configuration, role uses 'sqlite' per default but playbook sets up postgres by default
+matrix_mx_puppet_slack_database_engine: sqlite
+matrix_mx_puppet_slack_database_username: matrix_mx_puppet_slack
+matrix_mx_puppet_slack_database_password: ~
+matrix_mx_puppet_slack_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_slack_database_port: 5432
+matrix_mx_puppet_slack_database_name: matrix_mx_puppet_slack
+matrix_mx_puppet_slack_database_file: /data/database.db
+matrix_mx_puppet_slack_database_connString: >-2
+  {%- if matrix_mx_puppet_slack_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_slack_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_slack_database_engine }}://{{ matrix_mx_puppet_slack_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
index 3a0bca11144..bc722ee63b6 100644
--- a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
@@ -8,3 +8,23 @@
   with_items:
     - "matrix_mx_puppet_slack_appservice_token"
     - "matrix_mx_puppet_slack_homeserver_token"
+
+- block:
+  - name: Check if sqlite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_slack_data_path }}/database.db"
+    register: matrix_mx_puppet_slack_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mx_puppet_slack_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_slack_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_slack_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_slack_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_slack_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_slack_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_slack_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
index 7f1b21d785b..58c5c11abe2 100644
--- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
@@ -58,15 +58,18 @@ provisioning:
   apiPrefix: /_matrix/provision
 
 database:
+{% if matrix_mx_puppet_slack_database_engine == 'postgres' %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_slack:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_slack?sslmode=disable"
+  connString: {{ matrix_mx_puppet_slack_database_connString | to_json }}
+{% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  #filename: /data/database.db
+  filename: {{ matrix_mx_puppet_slack_database_file }}
+{% endif %}
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
index b5eb8473115..be343db45ab 100644
--- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
@@ -55,6 +55,20 @@ matrix_mx_puppet_steam_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_steam_login_shared_secret: ''
 
+matrix_mx_puppet_steam_database_engine: sqlite
+matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam
+matrix_mx_puppet_steam_database_password: ~
+matrix_mx_puppet_steam_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_steam_database_port: 5432
+matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam
+matrix_mx_puppet_steam_database_file: /data/database.db
+matrix_mx_puppet_steam_database_connString: >-2
+  {%- if matrix_mx_puppet_steam_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_steam_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_steam_database_engine }}://{{ matrix_mx_puppet_steam_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
index a8bc6a420b0..50ce15d9de0 100644
--- a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
@@ -8,3 +8,25 @@
   with_items:
     - "matrix_mx_puppet_steam_appservice_token"
     - "matrix_mx_puppet_steam_homeserver_token"
+
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_steam_data_path }}"
+    register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mx_puppet_steam_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_steam_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_steam_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_steam_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_steam_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_steam_database_engine == 'postgres'"
+
+
diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
index 14f86319612..8eda278dd52 100644
--- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
@@ -61,15 +61,18 @@ selfService:
   blacklist: {{ matrix_mx_puppet_steam_provisioning_blacklist|to_json }}
 
 database:
+{% if matrix_mx_puppet_steam_database_engine == 'postgres' %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_steam:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_steam') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_steam?sslmode=disable"
+  connString: {{ matrix_mx_puppet_steam_database_connString | to_json }}
+{% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  #filename: /data/database.db
+  filename: {{ matrix_mx_puppet_steam_database_file }}
+{% endif %}
 
 logging:
   # Log level of console output
diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
index 13438a21f28..f536029c0fe 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
@@ -60,6 +60,21 @@ matrix_mx_puppet_twitter_homeserver_token: ''
 # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth).
 matrix_mx_puppet_twitter_login_shared_secret: ''
 
+# Database configuration
+matrix_mx_puppet_twitter_database_engine: sqlite
+matrix_mx_puppet_twitter_database_username: matrix:mx_puppet_twitter
+matrix_mx_puppet_twitter_database_password: ~
+matrix_mx_puppet_twitter_database_hostname: 'matrix-postgres'
+matrix_mx_puppet_twitter_database_port: 5432
+matrix_mx_puppet_twitter_database_name: matrix_mx_puppet_twitter
+matrix_mx_puppet_twitter_database_file: /data/database.db
+matrix_mx_puppet_twitter_database_connString: >-2
+  {%- if matrix_mx_puppet_twitter_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable
+  {%- else if matrix_mx_puppet_twitter_database_engine == 'sqlite' -%}
+  {{ matrix_mx_puppet_twitter_database_engine }}://{{ matrix_mx_puppet_twitter_database_file }}
+  {%- endif -%}
+
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
index d13a39e1e90..0b3bd8f82d7 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
@@ -8,3 +8,25 @@
   with_items:
     - "matrix_mx_puppet_twitter_appservice_token"
     - "matrix_mx_puppet_twitter_homeserver_token"
+
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_mx_puppet_twitter_sqlite_data_path }}/database.db"
+    register: matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_mx_puppet_twitter_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_twitter_data_path }}/database.db.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_twitter_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_twitter_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_twitter_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_mx_puppet_twitter_database_engine == 'postgres'"
+
+
diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
index 853d23fc26c..cecb171d9e9 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
@@ -54,15 +54,18 @@ provisioning:
   apiPrefix: /_matrix/provision
 
 database:
+{% if matrix_mx_puppet_twitter_database_engine == 'postgres' %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: "postgres://matrix_puppet_twitter:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_puppet_twitter') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_puppet_twitter?sslmode=disable"
+  connString: {{ matrix_mx_puppet_twitter_database_connString | to_json }}
+{% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  #filename: /data/database.db
+  filename: {{ matrix_mx_puppet_twitter_database_file }}
+{% endif %}
 
 logging:
   # Log level of console output

From b9a04a7f953e504a3e6798eeebcc786f51bf9788 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 14:42:18 +0200
Subject: [PATCH 36/79] Rename some remaining `matrix_*_postgres_*` vars back
 to `matrix_*_database_*`

Looks like there are some that I missed in 087dbe4ddc80ba6
---
 group_vars/matrix_servers                     | 28 +++++++++----------
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  6 ++--
 .../tasks/setup_install.yml                   |  2 +-
 .../defaults/main.yml                         |  6 ++--
 .../tasks/setup_install.yml                   |  2 +-
 .../defaults/main.yml                         |  6 ++--
 .../tasks/setup_install.yml                   |  2 +-
 .../defaults/main.yml                         |  6 ++--
 .../tasks/setup_install.yml                   |  2 +-
 10 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index acf133e128d..ad1e8ce3204 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -953,37 +953,37 @@ matrix_postgres_additional_databases: |
       'name': matrix_appservice_discord_database_db_name,
       'username': matrix_appservice_discord_database_username,
       'password': matrix_appservice_discord_database_password,
-    }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_appservice_slack_database_db_name,
       'username': matrix_appservice_slack_database_username,
       'password': matrix_appservice_slack_database_password,
-    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_appservice_irc_database_db_name,
       'username': matrix_appservice_irc_database_username,
       'password': matrix_appservice_irc_database_password,
-    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_facebook_database_db_name,
       'username': matrix_mautrix_facebook_database_username,
       'password': matrix_mautrix_facebook_database_password,
-    }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_hangouts_database_db_name,
       'username': matrix_mautrix_hangouts_database_username,
       'password': matrix_mautrix_hangouts_database_password,
-    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_telegram_database_db_name,
       'username': matrix_mautrix_telegram_database_username,
       'password': matrix_mautrix_telegram_database_password,
-    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mautrix_whatsapp_database_db_name,
       'username': matrix_mautrix_whatsapp_database_username,
       'password': matrix_mautrix_whatsapp_database_password,
-    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
@@ -993,45 +993,45 @@ matrix_postgres_additional_databases: |
       'name': matrix_mx_puppet_skype_database_db_name,
       'username': matrix_mx_puppet_skype_database_username,
       'password': matrix_mx_puppet_skype_database_password,
-    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
     + ([{
       'name': matrix_mx_puppet_slack_database_db_name,
       'username': matrix_mx_puppet_slack_database_username,
       'password': matrix_mx_puppet_slack_database_password,
-    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_twitter_database_db_name,
       'username': matrix_mx_puppet_twitter_database_username,
       'password': matrix_mx_puppet_twitter_database_password,
-    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_postgres_hostname == 'matrix-postgres') else [])
+    }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_instagram_database_db_name,
       'username': matrix_mx_puppet_instagram_database_username,
       'password': matrix_mx_puppet_instagram_database_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
-           and matrix_mx_puppet_instagram_postgres_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_discord_database_db_name,
       'username': matrix_mx_puppet_discord_database_username,
       'password': matrix_mx_puppet_discord_database_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
-           and matrix_mx_puppet_discord_postgres_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_mx_puppet_steam_database_db_name,
       'username': matrix_mx_puppet_steam_database_username,
       'password': matrix_mx_puppet_steam_database_password,
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
-           and matrix_mx_puppet_steam_postgres_hostname == 'matrix-postgres') else [])
+           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
      + ([{
       'name': matrix_dimension_database_db_name,
       'username': matrix_dimension_database_username,
       'password': matrix_dimension_database_password,
     }] if (matrix_dimension_enabled
            and matrix_dimension_database_engine == 'postgres'
-           and matrix_dimension_postgres_hostname == 'matrix-postgres') else [])
+           and matrix_dimension_database_hostname == 'matrix-postgres') else [])
    }}
 
 ######################################################################
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 57a888671e4..e2f97b8532e 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -55,14 +55,14 @@ matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db"
 
 matrix_appservice_discord_database_username: 'matrix_appservice_discord'
 matrix_appservice_discord_database_password: 'some-password'
-matrix_appservice_discord_postgres_hostname: 'matrix-postgres'
+matrix_appservice_discord_database_hostname: 'matrix-postgres'
 matrix_appservice_discord_database_port: 5432
 matrix_appservice_discord_database_db_name: 'matrix_appservice_discord'
 
 # These 2 variables are what actually ends up in the bridge configuration.
 # It's best if you don't change them directly, but rather redefine the sub-variables that constitute them.
 matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}"
-matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_postgres_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}'
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}'
 
 
 # Tells whether the bot should make use of "Privileged Gateway Intents".
diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index 246c32fe683..c02197a5b37 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -47,16 +47,16 @@ matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebo
 
 matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook'
 matrix_mautrix_facebook_database_password: 'some-password'
-matrix_mautrix_facebook_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_facebook_database_hostname: 'matrix-postgres'
 matrix_mautrix_facebook_database_port: 5432
 matrix_mautrix_facebook_database_db_name: 'matrix_mautrix_facebook'
 
-matrix_mautrix_facebook_postgres_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_postgres_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}'
+matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}'
 
 matrix_mautrix_facebook_appservice_database: "{{
 	{
 		'sqlite': ('sqlite://' + matrix_mautrix_facebook_sqlite_database_path_in_container),
-		'postgres': matrix_mautrix_facebook_postgres_connection_string,
+		'postgres': matrix_mautrix_facebook_database_connection_string,
 	}[matrix_mautrix_facebook_database_engine]
 }}"
 
diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
index c3c8d61fee6..1a0ff70cb03 100644
--- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
@@ -23,7 +23,7 @@
         To continue using SQLite, opt into it explicitly: add `matrix_mautrix_facebook_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_postgres_connection_string'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_database_connection_string'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mautrix_facebook_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
index 04280ac42a6..5d38289ba1c 100644
--- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
@@ -54,16 +54,16 @@ matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangou
 
 matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts'
 matrix_mautrix_hangouts_database_password: 'some-password'
-matrix_mautrix_hangouts_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_hangouts_database_hostname: 'matrix-postgres'
 matrix_mautrix_hangouts_database_port: 5432
 matrix_mautrix_hangouts_database_db_name: 'matrix_mautrix_hangouts'
 
-matrix_mautrix_hangouts_postgres_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_postgres_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}'
+matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}'
 
 matrix_mautrix_hangouts_appservice_database: "{{
 	{
 		'sqlite': ('sqlite://' + matrix_mautrix_hangouts_sqlite_database_path_in_container),
-		'postgres': matrix_mautrix_hangouts_postgres_connection_string,
+		'postgres': matrix_mautrix_hangouts_database_connection_string,
 	}[matrix_mautrix_hangouts_database_engine]
 }}"
 
diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
index 9e7bfbba9ba..02c34357ced 100644
--- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
@@ -23,7 +23,7 @@
         To continue using SQLite, opt into it explicitly: add `matrix_mautrix_hangouts_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_postgres_connection_string'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_database_connection_string'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mautrix_hangouts_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index 4f584128e4e..fcdf95db295 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -58,16 +58,16 @@ matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegr
 
 matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram'
 matrix_mautrix_telegram_database_password: 'some-password'
-matrix_mautrix_telegram_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_telegram_database_hostname: 'matrix-postgres'
 matrix_mautrix_telegram_database_port: 5432
 matrix_mautrix_telegram_database_db_name: 'matrix_mautrix_telegram'
 
-matrix_mautrix_telegram_postgres_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_postgres_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}'
+matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}'
 
 matrix_mautrix_telegram_appservice_database: "{{
 	{
 		'sqlite': ('sqlite://' + matrix_mautrix_telegram_sqlite_database_path_in_container),
-		'postgres': matrix_mautrix_telegram_postgres_connection_string,
+		'postgres': matrix_mautrix_telegram_database_connection_string,
 	}[matrix_mautrix_telegram_database_engine]
 }}"
 
diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
index db1a8f832c8..d898c5908c9 100644
--- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
@@ -23,7 +23,7 @@
         To continue using SQLite, opt into it explicitly: add `matrix_mautrix_telegram_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_postgres_connection_string'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_database_connection_string'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mautrix_telegram_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
index 357f5892599..7e198b36883 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
@@ -42,11 +42,11 @@ matrix_mautrix_whatsapp_sqlite_database_path_in_container: "/data/mautrix-whatsa
 
 matrix_mautrix_whatsapp_database_username: 'matrix_mautrix_whatsapp'
 matrix_mautrix_whatsapp_database_password: 'some-password'
-matrix_mautrix_whatsapp_postgres_hostname: 'matrix-postgres'
+matrix_mautrix_whatsapp_database_hostname: 'matrix-postgres'
 matrix_mautrix_whatsapp_database_port: 5432
 matrix_mautrix_whatsapp_database_db_name: 'matrix_mautrix_whatsapp'
 
-matrix_mautrix_whatsapp_postgres_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_postgres_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}'
+matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}'
 
 matrix_mautrix_whatsapp_appservice_database_type: "{{
 	{
@@ -58,7 +58,7 @@ matrix_mautrix_whatsapp_appservice_database_type: "{{
 matrix_mautrix_whatsapp_appservice_database_uri: "{{
 	{
 		'sqlite': matrix_mautrix_whatsapp_sqlite_database_path_in_container,
-		'postgres': matrix_mautrix_whatsapp_postgres_connection_string,
+		'postgres': matrix_mautrix_whatsapp_database_connection_string,
 	}[matrix_mautrix_whatsapp_database_engine]
 }}"
 
diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
index 15453e8caf0..9521e0a53fc 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
@@ -23,7 +23,7 @@
         To continue using SQLite, opt into it explicitly: add `matrix_mautrix_whatsapp_database_engine: sqlite` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing SQLite database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_postgres_connection_string'`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_database_connection_string'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mautrix_whatsapp_database_engine == 'postgres'"

From aa828ff9f64415a404b3d7bb2a70690fea968d02 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 14:50:04 +0200
Subject: [PATCH 37/79] Separate matrix-reminder-bot install/uninstall tasks

---
 .../tasks/main.yml                            |  9 +-
 .../tasks/setup.yml                           | 88 -------------------
 .../tasks/setup_install.yml                   | 41 +++++++++
 .../tasks/setup_uninstall.yml                 | 35 ++++++++
 4 files changed, 84 insertions(+), 89 deletions(-)
 delete mode 100644 roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml
 create mode 100644 roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
 create mode 100644 roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml

diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml
index ade3d19119a..fc2afddb278 100644
--- a/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml
@@ -8,7 +8,14 @@
     - setup-all
     - setup-bot-matrix-reminder-bot
 
-- import_tasks: "{{ role_path }}/tasks/setup.yml"
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_bot_matrix_reminder_bot_enabled|bool"
+  tags:
+    - setup-all
+    - setup-bot-matrix-reminder-bot
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_bot_matrix_reminder_bot_enabled|bool"
   tags:
     - setup-all
     - setup-bot-matrix-reminder-bot
diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml
deleted file mode 100644
index bc211e3103e..00000000000
--- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml
+++ /dev/null
@@ -1,88 +0,0 @@
----
-
-#
-# Tasks related to setting up matrix-reminder-bot
-#
-
-- name: Ensure matrix-reminder-bot paths exist
-  file:
-    path: "{{ item.path }}"
-    state: directory
-    mode: 0750
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_groupname }}"
-  with_items:
-    - { path: "{{ matrix_bot_matrix_reminder_bot_config_path }}", when: true }
-    - { path: "{{ matrix_bot_matrix_reminder_bot_data_path }}", when: true }
-    - { path: "{{ matrix_bot_matrix_reminder_bot_data_store_path }}", when: true }
-  when: matrix_bot_matrix_reminder_bot_enabled|bool and item.when
-
-- name: Ensure matrix-reminder-bot image is pulled
-  docker_image:
-    name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_matrix_reminder_bot_docker_image_force_pull }}"
-  when: matrix_bot_matrix_reminder_bot_enabled|bool
-
-- name: Ensure matrix-reminder-bot config installed
-  copy:
-    content: "{{ matrix_bot_matrix_reminder_bot_configuration|to_nice_yaml }}"
-    dest: "{{ matrix_bot_matrix_reminder_bot_config_path }}/config.yaml"
-    mode: 0644
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_groupname }}"
-  when: matrix_bot_matrix_reminder_bot_enabled|bool
-
-- name: Ensure matrix-matrix-reminder-bot.service installed
-  template:
-    src: "{{ role_path }}/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2"
-    dest: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
-    mode: 0644
-  register: matrix_bot_matrix_reminder_bot_systemd_service_result
-  when: matrix_bot_matrix_reminder_bot_enabled|bool
-
-- name: Ensure systemd reloaded after matrix-matrix-reminder-bot.service installation
-  service:
-    daemon_reload: yes
-  when: "matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_systemd_service_result.changed"
-
-#
-# Tasks related to getting rid of matrix-reminder-bot (if it was previously enabled)
-#
-
-- name: Check existence of matrix-matrix-reminder-bot service
-  stat:
-    path: "{{ matrix_systemd_path }}/matrix-matrix-reminder-bot.service"
-  register: matrix_bot_matrix_reminder_bot_service_stat
-
-- name: Ensure matrix-matrix-reminder-bot is stopped
-  service:
-    name: matrix-matrix-reminder-bot
-    state: stopped
-    daemon_reload: yes
-  register: stopping_result
-  when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists"
-
-- name: Ensure matrix-matrix-reminder-bot.service doesn't exist
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-matrix-reminder-bot.service"
-    state: absent
-  when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists"
-
-- name: Ensure systemd reloaded after matrix-matrix-reminder-bot.service removal
-  service:
-    daemon_reload: yes
-  when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists"
-
-- name: Ensure Matrix matrix-reminder-bot paths don't exist
-  file:
-    path: "{{ matrix_bot_matrix_reminder_bot_base_path }}"
-    state: absent
-  when: "not matrix_bot_matrix_reminder_bot_enabled|bool"
-
-- name: Ensure matrix-reminder-bot Docker image doesn't exist
-  docker_image:
-    name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}"
-    state: absent
-  when: "not matrix_bot_matrix_reminder_bot_enabled|bool"
diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
new file mode 100644
index 00000000000..366c39c38ba
--- /dev/null
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
@@ -0,0 +1,41 @@
+---
+
+- name: Ensure matrix-reminder-bot paths exist
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - { path: "{{ matrix_bot_matrix_reminder_bot_config_path }}", when: true }
+    - { path: "{{ matrix_bot_matrix_reminder_bot_data_path }}", when: true }
+    - { path: "{{ matrix_bot_matrix_reminder_bot_data_store_path }}", when: true }
+  when: "item.when|bool"
+
+- name: Ensure matrix-reminder-bot image is pulled
+  docker_image:
+    name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_matrix_reminder_bot_docker_image_force_pull }}"
+
+- name: Ensure matrix-reminder-bot config installed
+  copy:
+    content: "{{ matrix_bot_matrix_reminder_bot_configuration|to_nice_yaml }}"
+    dest: "{{ matrix_bot_matrix_reminder_bot_config_path }}/config.yaml"
+    mode: 0644
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+
+- name: Ensure matrix-bot-matrix-reminder-bot.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
+    mode: 0644
+  register: matrix_bot_matrix_reminder_bot_systemd_service_result
+
+- name: Ensure systemd reloaded after matrix-bot-matrix-reminder-bot.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_bot_matrix_reminder_bot_systemd_service_result.changed|bool"
diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml
new file mode 100644
index 00000000000..744f474d113
--- /dev/null
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml
@@ -0,0 +1,35 @@
+---
+
+- name: Check existence of matrix-matrix-reminder-bot service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
+  register: matrix_bot_matrix_reminder_bot_service_stat
+
+- name: Ensure matrix-matrix-reminder-bot is stopped
+  service:
+    name: matrix-matrix-reminder-bot
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-bot-matrix-reminder-bot.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service"
+    state: absent
+  when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-bot-matrix-reminder-bot.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool"
+
+- name: Ensure Matrix matrix-reminder-bot paths don't exist
+  file:
+    path: "{{ matrix_bot_matrix_reminder_bot_base_path }}"
+    state: absent
+
+- name: Ensure matrix-reminder-bot Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}"
+    state: absent

From dde1c9f899230813ca70e657c4d7873bb890f6e4 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 14:53:35 +0200
Subject: [PATCH 38/79] Fix indentation causing YAML syntax error

---
 .../tasks/validate_config.yml                                  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
index 7000a8a175b..c2115cbb2e2 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
@@ -15,7 +15,7 @@
       path: "{{ matrix_mx_puppet_instagram_data_path }}/database.db"
     register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result
 
-   - name: Fail if an SQLite database already exists when using Postgres
+  - name: Fail if an SQLite database already exists when using Postgres
     fail:
       msg: >-
         matrix_mx_puppet_instagram_database_engine has been set to `postgres` (which is our new default now).
@@ -28,4 +28,3 @@
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists"
   when: "matrix_mx_puppet_instagram_database_engine == 'postgres'"
-

From e3a0c9addac73580da23f1e386a67ba14d7461bf Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 15:02:11 +0200
Subject: [PATCH 39/79] Add (Postgres + SQLite) support to matrix-reminder-bot

This has been tested and appears to work.
---
 group_vars/matrix_servers                     | 10 +++++++
 .../defaults/main.yml                         | 28 +++++++++++++++++++
 .../tasks/setup_install.yml                   | 20 +++++++++++++
 .../templates/config.yaml.j2                  |  2 +-
 4 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index ad1e8ce3204..d6d6cc4a07b 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -580,6 +580,10 @@ matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key
 # We don't enable bots by default.
 matrix_bot_matrix_reminder_bot_enabled: false
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_bot_matrix_reminder_bot_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_bot_matrix_reminder_bot_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'reminder.bot.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-bot-matrix-reminder-bot
@@ -949,6 +953,12 @@ matrix_postgres_db_name: "homeserver"
 
 matrix_postgres_additional_databases: |
   {{
+    ([{
+      'name': matrix_bot_matrix_reminder_bot_database_db_name,
+      'username': matrix_bot_matrix_reminder_bot_database_username,
+      'password': matrix_bot_matrix_reminder_bot_database_password,
+    }] if (matrix_bot_matrix_reminder_bot_enabled and matrix_bot_matrix_reminder_bot_database_engine == 'postgres' and matrix_bot_matrix_reminder_bot_database_hostname == 'matrix-postgres') else [])
+    +
     ([{
       'name': matrix_appservice_discord_database_db_name,
       'username': matrix_appservice_discord_database_username,
diff --git a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
index 3302814732c..70e7b2d2fb8 100644
--- a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
@@ -21,6 +21,34 @@ matrix_bot_matrix_reminder_bot_systemd_required_services_list: ['docker.service'
 matrix_bot_matrix_reminder_bot_systemd_wanted_services_list: []
 
 
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_bot_matrix_reminder_bot_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_bot_matrix_reminder_bot_database_*` variables
+matrix_bot_matrix_reminder_bot_database_engine: 'sqlite'
+
+matrix_bot_matrix_reminder_bot_sqlite_database_path_local: "{{ matrix_bot_matrix_reminder_bot_data_path }}/bot.db"
+matrix_bot_matrix_reminder_bot_sqlite_database_path_in_container: "/data/bot.db"
+
+matrix_bot_matrix_reminder_bot_database_username: 'matrix_reminder_bot'
+matrix_bot_matrix_reminder_bot_database_password: 'some-password'
+matrix_bot_matrix_reminder_bot_database_hostname: 'matrix-postgres'
+matrix_bot_matrix_reminder_bot_database_port: 5432
+matrix_bot_matrix_reminder_bot_database_db_name: 'matrix_reminder_bot'
+
+matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_db_name }}'
+
+matrix_bot_matrix_reminder_bot_storage_database: "{{
+	{
+		'sqlite': ('sqlite://' + matrix_bot_matrix_reminder_bot_sqlite_database_path_in_container),
+		'postgres': matrix_bot_matrix_reminder_bot_database_connection_string,
+	}[matrix_bot_matrix_reminder_bot_database_engine]
+}}"
+
+
 # The bot's username. This user needs to be created manually beforehand.
 # Also see `matrix_bot_matrix_reminder_bot_user_password`.
 matrix_bot_matrix_reminder_bot_matrix_user_id_localpart: "bot.matrix-reminder-bot"
diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
index 366c39c38ba..4fbafa3adb8 100644
--- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
@@ -1,5 +1,25 @@
 ---
 
+- block:
+  - name: Check if an SQLite database already exists
+    stat:
+      path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}"
+    register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result
+
+  - name: Fail if an SQLite database already exists when using Postgres
+    fail:
+      msg: >-
+        matrix_bot_matrix_reminder_bot_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing SQLite database in {{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}.
+        It appears that you've been using this bridge with the SQLite engine until now.
+        To continue using SQLite, opt into it explicitly: add `matrix_bot_matrix_reminder_bot_database_engine: sqlite` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing SQLite database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_bot_matrix_reminder_bot_database_connection_string'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists"
+  when: "matrix_bot_matrix_reminder_bot_database_engine == 'postgres'"
+
 - name: Ensure matrix-reminder-bot paths exist
   file:
     path: "{{ item.path }}"
diff --git a/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2 b/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2
index e5ba5651727..59643958468 100644
--- a/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2
+++ b/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2
@@ -23,7 +23,7 @@ storage:
   # For Postgres, this would look like:
   #     database: "postgres://username:password@localhost/dbname?sslmode=disable"
   #database: "postgres://matrix-reminder-bot:remindme@localhost/matrix-reminder-bot?sslmode=disable"
-  database: "sqlite:///data/bot.db"
+  database: {{ matrix_bot_matrix_reminder_bot_storage_database|to_json }}
   # The path to a directory for internal bot storage
   # containing encryption keys, sync tokens, etc.
   store_path: "/data/store"

From 54da61f81b56e4c5906d09cd7dbbf0fb35092948 Mon Sep 17 00:00:00 2001
From: transcaffeine <transcaffeine@finallycoffee.eu>
Date: Mon, 14 Dec 2020 14:02:51 +0100
Subject: [PATCH 40/79] add postgres support mx-appservice-[slack|irc] with
 fallback to nedb in role and migration notice

---
 .../defaults/main.yml                         | 14 ++++++++++++
 .../tasks/validate_config.yml                 | 20 +++++++++++++++++
 .../templates/config.yaml.j2                  |  4 ++--
 .../defaults/main.yml                         | 15 +++++++++++++
 .../tasks/validate_config.yml                 | 22 ++++++++++++++++++-
 .../templates/config.yaml.j2                  |  6 +++--
 6 files changed, 76 insertions(+), 5 deletions(-)

diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml
index 7ff2d46a37c..a1c762537fb 100644
--- a/roles/matrix-bridge-appservice-irc/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml
@@ -16,6 +16,20 @@ matrix_appservice_irc_homeserver_domain: '{{ matrix_domain }}'
 matrix_appservice_irc_homeserver_enablePresence: true
 matrix_appservice_irc_appservice_address: 'http://matrix-appservice-irc:9999'
 
+matrix_appservice_irc_database_engine: nedb
+matrix_appservice_irc_database_username: matrix_appservice_irc
+matrix_appservice_irc_database_password: ~
+matrix_appservice_irc_database_hostname: 'matrix-postgres'
+matrix_appservice_irc_database_port: 5432
+matrix_appservice_irc_database_name: matrix_appservice_irc
+matrix_appservice_irc_databasa
+matrix_appservice_irc_database_connString: >-2
+  {%- if matrix_appservice_irc_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable
+  {%- else if matrix_appservice_irc_database_engine == 'nedb' -%}
+  {{ matrix_appservice_irc_database_engine }}://{{ matrix_appservice_irc_database_file }}
+  {%- endif -%}
+
 matrix_appservice_irc_ircService_servers: []
 
 # Example of `matrix_appservice_irc_ircService_servers` with one server (and all its options):
diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
index bd08427c2d0..e0e932bcc14 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
@@ -33,3 +33,23 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_appservice_irc_container_expose_client_server_api_port', 'new': '<superseded by matrix_appservice_irc_container_http_host_bind_port>'}
+
+- block:
+  - name: Check if a neDB database already exists
+    stat:
+      path: "{{ matrix_appservice_irc_data_path }}/"
+    register: matrix_appservice_irc_nedb_stat_result
+
+  - name: Fail if an neDB database already exists when using Postgres
+    fail:
+      msg: >-2
+        matrix_appservice_irc_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing neDB database in {{ matrix_appservice_irc_data_path }}/.
+        It appears that you've been using this bridge with the neDB engine until now.
+        To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing neDB database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_appservice_irc_nedb_stat_result.stat.exists"
+  when: "matrix_appservice_irc_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
index 83b71835872..0da28403f91 100644
--- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
@@ -127,8 +127,8 @@ advanced:
 # Use an external database to store bridge state.
 database:
   # database engine (must be 'postgres' or 'nedb'). Default: nedb
-  engine: "postgres"
+  engine: {{ matrix_appservice_irc_database_engine }}
   # Either a PostgreSQL connection string, or a path to the NeDB storage directory.
   # For postgres, it must start with postgres://
   # For NeDB, it must start with nedb://. The path is relative to the project directory.
-  connectionString: "postgres://matrix_appservice_irc:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_irc') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_irc"
+  connectionString: {{ matrix_appservice_irc_database_connString | to_json }}
diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml
index f2cfd8f9361..e2127c29a81 100644
--- a/roles/matrix-bridge-appservice-slack/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml
@@ -45,6 +45,21 @@ matrix_appservice_slack_appservice_token: ''
 matrix_appservice_slack_homeserver_token: ''
 matrix_appservice_slack_id_token: ''
 
+matrix_appservice_slack_database_engine: nedb
+matrix_appservice_slack_database_username: matrix_appservice_slack
+matrix_appservice_slack_database_password: ~
+matrix_appservice_slack_database_hostname: 'matrix-postgres'
+matrix_appservice_slack_database_port: 5432
+matrix_appservice_slack_database_name: matrix_appservice_slack
+matrix_appservice_slack_database_file: /data
+matrix_appservice_slack_database_connString: >-2
+  {%- if matrix_appservice_slack_database_engine == 'postgres' -%}
+  postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable
+  {%- else if matrix_appservice_slack_database_engine == 'nedb' -%}
+  {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }}
+  {%- endif -%}
+
+
 matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
 
 matrix_appservice_slack_configuration_extension_yaml: |
diff --git a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
index 5da5f947e1e..5421b112a8d 100644
--- a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
@@ -9,4 +9,24 @@
     - "matrix_appservice_slack_control_room_id"
     - "matrix_appservice_slack_appservice_token"
     - "matrix_appservice_slack_homeserver_token"
-    - "matrix_appservice_slack_id_token"
\ No newline at end of file
+    - "matrix_appservice_slack_id_token"
+
+- block:
+  - name: Check if a neDB database already exists
+    stat:
+      path: "{{ matrix_appservice_slack_data_path }}/"
+    register: matrix_appservice_slack_nedb_stat_result
+
+  - name: Fail if an neDB database already exists when using Postgres
+    fail:
+      msg: >-2
+        matrix_appservice_slack_database_engine has been set to `postgres` (which is our new default now).
+        However, we've discovered an existing neDB database in {{ matrix_appservice_slack_data_path }}/.
+        It appears that you've been using this bridge with the neDB engine until now.
+        To continue using neDB, opt into it explicitly: add `matrix_appservice_slack_database_engine: nedb` to your vars.yml file and re-run this same command.
+        Alternatively, to migrate your existing neDB database to Postgres:
+        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
+        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_slack_data_path }} postgres_connection_string_variable_name=matrix_appservice_slack_database_connString'`)
+        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
+    when: "matrix_appservice_slack_nedb_stat_result.stat.exists"
+  when: "matrix_appservice_slack_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
index 6c491134af5..0ae137381b7 100644
--- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
@@ -9,9 +9,11 @@ homeserver:
     url: "{{ matrix_appservice_slack_homeserver_url }}"
     server_name: "{{ matrix_domain }}"
 
+{% if matrix_appservice_slack_database_engine == 'nedb' %}
 dbdir: "/data"
+{% endif %}
 db:
-    engine: "postgres"
-    connectionString: "postgresql://matrix_appservice_slack:{{ matrix_addtional_databases | selectattr('name', 'equalto', 'matrix_appservice_slack') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_appservice_slack"
+    engine: "{{ matrix_appservice_slack_database_engine }}"
+    connectionString: {{ matrix_appservice_slack_database_connString | to_json }}
 
 matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"

From bc376c2fb2f2da5172dfe6ef130317921a23070f Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 15:55:35 +0200
Subject: [PATCH 41/79] Add database migration utility to matrix-postgres role

---
 .../tasks/util/migrate_db_to_postgres.yml     | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100644 roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml

diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
new file mode 100644
index 00000000000..ec635f99c2d
--- /dev/null
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -0,0 +1,111 @@
+---
+
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate."
+  when: "not matrix_postgres_enabled|bool"
+
+- name: Fail if util called incorrectly (missing matrix_postgres_db_migration_request)
+  fail:
+    msg: "The `matrix_postgres_db_migration_request` variable needs to be provided to this util."
+  when: "matrix_postgres_db_migration_request is not defined"
+
+- name: Fail if util called incorrectly (invalid matrix_postgres_db_migration_request)
+  fail:
+    msg: "The `matrix_postgres_db_migration_request` variable needs to contain `{{ item }}`."
+  with_items:
+    - src
+    - dst
+    - caller
+    - engine_variable_name
+    - systemd_services_to_stop
+  when: "item not in matrix_postgres_db_migration_request"
+
+- name: Check if the provided source database file exists
+  stat:
+    path: "{{ matrix_postgres_db_migration_request.src }}"
+  register: matrix_postgres_db_migration_request_src_stat_result
+
+- name: Fail if provided source database file doesn't exist
+  fail:
+    msg: "File cannot be found on the server at {{ matrix_postgres_db_migration_request.src }}"
+  when: "not matrix_postgres_db_migration_request_src_stat_result.stat.exists"
+
+- name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }})
+  fail:
+    msg: >-
+      {{ matrix_postgres_db_migration_request.engine_variable_name }} has been set to `postgres` (which is our new default now).
+      However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old_default }}) in {{ matrix_postgres_db_migration_request.src }}.
+      It appears that you've been using this bridge with a file-based database engine until now.
+      To continue using {{ matrix_postgres_db_migration_request.engine_old_default }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old_default }}` to your vars.yml file and re-run this same command.
+      We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now).
+      Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740
+  when: "matrix_architecture != 'amd64'"
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+# Actual import work
+
+# matrix-postgres is most likely started already
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+# We only stop services here, leaving it to the caller to start them later.
+#
+# We can't start them, because they probably need to be reconfigured too (changing the configuration from using SQLite to Postgres, etc.),
+# before starting.
+#
+# Since the caller will be starting them, it might make sense to leave stopping to it as well.
+# However, we don't do it, because it's simpler having it here, and it also gets to happen only if we'll be doing an import.
+# If we bailed out (somewhere above), nothing would have gotten stopped. It's nice to leave this running in such cases.
+- name: Ensure systemd services stopped
+  service:
+    name: "{{ item }}"
+    state: stopped
+  with_items: "{{ matrix_postgres_db_migration_request.systemd_services_to_stop }}"
+
+- name: Import SQLite database from {{ matrix_postgres_db_migration_request.src }} into Postgres
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_postgres_db_migration_request.src }},dst=/in.db,ro
+      --entrypoint=/bin/sh
+      {{ matrix_postgres_pgloader_docker_image }}
+      -c
+      'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}'
+
+- name: Archive SQLite database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
+  command:
+    cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup"
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your {{ matrix_postgres_db_migration_request.engine_variable_name }} database file has been imported into Postgres. The original file has been moved from `{{ matrix_postgres_db_migration_request.src }}` to `{{ matrix_postgres_db_migration_request.src }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
+        ]
+      }}

From dc7850e83cc0d23f0f5b6085e8e974f6e89f0e0e Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:03:03 +0200
Subject: [PATCH 42/79] Fix wording and variable names a bit

---
 .../tasks/util/migrate_db_to_postgres.yml            | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index ec635f99c2d..60539a91b05 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -35,9 +35,9 @@
   fail:
     msg: >-
       {{ matrix_postgres_db_migration_request.engine_variable_name }} has been set to `postgres` (which is our new default now).
-      However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old_default }}) in {{ matrix_postgres_db_migration_request.src }}.
+      However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}.
       It appears that you've been using this bridge with a file-based database engine until now.
-      To continue using {{ matrix_postgres_db_migration_request.engine_old_default }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old_default }}` to your vars.yml file and re-run this same command.
+      To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command.
       We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now).
       Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740
   when: "matrix_architecture != 'amd64'"
@@ -75,13 +75,13 @@
 # Since the caller will be starting them, it might make sense to leave stopping to it as well.
 # However, we don't do it, because it's simpler having it here, and it also gets to happen only if we'll be doing an import.
 # If we bailed out (somewhere above), nothing would have gotten stopped. It's nice to leave this running in such cases.
-- name: Ensure systemd services stopped
+- name: Ensure systemd services blocking the database import are stopped
   service:
     name: "{{ item }}"
     state: stopped
   with_items: "{{ matrix_postgres_db_migration_request.systemd_services_to_stop }}"
 
-- name: Import SQLite database from {{ matrix_postgres_db_migration_request.src }} into Postgres
+- name: Import {{ matrix_postgres_db_migration_request.engine_old }} database from {{ matrix_postgres_db_migration_request.src }} into Postgres
   command:
     cmd: >-
       {{ matrix_host_command_docker }} run
@@ -95,7 +95,7 @@
       -c
       'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}'
 
-- name: Archive SQLite database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
+- name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
   command:
     cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup"
 
@@ -106,6 +106,6 @@
         matrix_playbook_runtime_results|default([])
         +
         [
-          "NOTE: Your {{ matrix_postgres_db_migration_request.engine_variable_name }} database file has been imported into Postgres. The original file has been moved from `{{ matrix_postgres_db_migration_request.src }}` to `{{ matrix_postgres_db_migration_request.src }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
+          "NOTE: Your {{ matrix_postgres_db_migration_request.engine_old }} database file has been imported into Postgres. The original database file has been moved from `{{ matrix_postgres_db_migration_request.src }}` to `{{ matrix_postgres_db_migration_request.src }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
         ]
       }}

From ac37091d01c4b38f59c1a57c7b83305bb335ae6d Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:03:32 +0200
Subject: [PATCH 43/79] Enable automatic (SQLite -> Postgres) migration for
 matrix-reminder-bot

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
index 4fbafa3adb8..fea1e00ddff 100644
--- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml
@@ -1,23 +1,29 @@
 ---
 
+- set_fact:
+    matrix_bot_matrix_reminder_bot_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}"
-    register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_bot_matrix_reminder_bot_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_bot_matrix_reminder_bot_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_bot_matrix_reminder_bot_database_connection_string'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}"
+      register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}"
+              dst: "{{ matrix_bot_matrix_reminder_bot_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_bot_matrix_reminder_bot_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-bot-matrix-reminder-bot.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_bot_matrix_reminder_bot_requires_restart: true
+      when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_bot_matrix_reminder_bot_database_engine == 'postgres'"
 
 - name: Ensure matrix-reminder-bot paths exist
@@ -59,3 +65,9 @@
   service:
     daemon_reload: yes
   when: "matrix_bot_matrix_reminder_bot_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-bot-matrix-reminder-bot.service restarted, if necessary
+  service:
+    name: "matrix-bot-matrix-reminder-bot.service"
+    state: restarted
+  when: "matrix_bot_matrix_reminder_bot_requires_restart|bool"

From c1431b28f0384002843b5a059f4fe11caa3d8a57 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:13:57 +0200
Subject: [PATCH 44/79] Make use of matrix_postgres_db_migration_request.caller

---
 roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index 60539a91b05..b77ce4b00c3 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -34,7 +34,7 @@
 - name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }})
   fail:
     msg: >-
-      {{ matrix_postgres_db_migration_request.engine_variable_name }} has been set to `postgres` (which is our new default now).
+      {{ matrix_postgres_db_migration_request.engine_variable_name }} (part of {{ matrix_postgres_db_migration_request.caller }}) has been set to `postgres` (which is our new default now).
       However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}.
       It appears that you've been using this bridge with a file-based database engine until now.
       To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command.

From bbc08722c5ba4c02e39f747e0631696022316321 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:14:23 +0200
Subject: [PATCH 45/79] Enable automatic (SQLite -> Postgres) migration for
 matrix-mautrix-facebook

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
index 1a0ff70cb03..7f3104467fb 100644
--- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml
@@ -8,24 +8,30 @@
       The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- set_fact:
+    matrix_mautrix_facebook_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}"
-    register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mautrix_facebook_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mautrix_facebook_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_facebook_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_facebook_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_facebook_database_connection_string'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}"
+      register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}"
+              dst: "{{ matrix_mautrix_facebook_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mautrix_facebook_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mautrix-facebook.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mautrix_facebook_requires_restart: true
+      when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_mautrix_facebook_database_engine == 'postgres'"
 
 - name: Ensure Mautrix Facebook image is pulled
@@ -114,3 +120,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mautrix_facebook_systemd_service_result.changed"
+
+- name: Ensure matrix-mautrix-facebook.service restarted, if necessary
+  service:
+    name: "matrix-mautrix-facebook.service"
+    state: restarted
+  when: "matrix_mautrix_facebook_requires_restart|bool"

From 3ba852026663407c0db3707ee9a659e850186728 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:18:38 +0200
Subject: [PATCH 46/79] Enable automatic (SQLite -> Postgres) migration for
 matrix-mautrix-hangouts

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
index 02c34357ced..15af96265a4 100644
--- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml
@@ -8,24 +8,30 @@
       The matrix-bridge-mautrix-hangouts role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- set_fact:
+    matrix_mautrix_hangouts_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}"
-    register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mautrix_hangouts_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mautrix_hangouts_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_hangouts_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_hangouts_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_hangouts_database_connection_string'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}"
+      register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}"
+              dst: "{{ matrix_mautrix_hangouts_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mautrix_hangouts_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mautrix-hangouts.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mautrix_hangouts_requires_restart: true
+      when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_mautrix_hangouts_database_engine == 'postgres'"
 
 - name: Ensure Mautrix Hangouts image is pulled
@@ -113,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mautrix_hangouts_systemd_service_result.changed"
+
+- name: Ensure matrix-mautrix-hangouts.service restarted, if necessary
+  service:
+    name: "matrix-mautrix-hangouts.service"
+    state: restarted
+  when: "matrix_mautrix_hangouts_requires_restart|bool"

From 86a8091768bbe8871212761c99de5aa734462039 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:19:54 +0200
Subject: [PATCH 47/79] Enable automatic (SQLite -> Postgres) migration for
 matrix-mautrix-telegram

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
index d898c5908c9..6a37974a833 100644
--- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml
@@ -8,24 +8,30 @@
       The matrix-bridge-mautrix-telegram role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- set_fact:
+    matrix_mautrix_telegram_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}"
-    register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mautrix_telegram_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mautrix_telegram_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_telegram_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_telegram_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_telegram_database_connection_string'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}"
+      register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}"
+              dst: "{{ matrix_mautrix_telegram_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mautrix_telegram_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mautrix-telegram.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mautrix_telegram_requires_restart: true
+      when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_mautrix_telegram_database_engine == 'postgres'"
 
 - name: Ensure Mautrix Telegram image is pulled
@@ -91,3 +97,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mautrix_telegram_systemd_service_result.changed"
+
+- name: Ensure matrix-mautrix-telegram.service restarted, if necessary
+  service:
+    name: "matrix-mautrix-telegram.service"
+    state: restarted
+  when: "matrix_mautrix_telegram_requires_restart|bool"

From 13f84e2ad588668f26010e8f8de07f35c6527e97 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:21:01 +0200
Subject: [PATCH 48/79] Enable automatic (SQLite -> Postgres) migration for
 matrix-mautrix-whatsapp

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
index 9521e0a53fc..8d894a8493a 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml
@@ -8,24 +8,30 @@
       The matrix-bridge-mautrix-whatsapp role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- set_fact:
+    matrix_mautrix_whatsapp_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}"
-    register: matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mautrix_whatsapp_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mautrix_whatsapp_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mautrix_whatsapp_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mautrix_whatsapp_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_mautrix_whatsapp_database_connection_string'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}"
+      register: matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}"
+              dst: "{{ matrix_mautrix_whatsapp_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mautrix_whatsapp_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mautrix-whatsapp.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mautrix_whatsapp_requires_restart: true
+      when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_mautrix_whatsapp_database_engine == 'postgres'"
 
 - name: Ensure Mautrix Whatsapp image is pulled
@@ -100,3 +106,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mautrix_whatsapp_systemd_service_result.changed"
+
+- name: Ensure matrix-mautrix-whatsapp.service restarted, if necessary
+  service:
+    name: "matrix-mautrix-whatsapp.service"
+    state: restarted
+  when: "matrix_mautrix_whatsapp_requires_restart|bool"

From cba973d6b555b9288e8791eecb601c4e80aba4f6 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:24:56 +0200
Subject: [PATCH 49/79] Enable automatic (SQLite -> Postgres) migration for
 matrix-appservice-discord

---
 .../tasks/setup_install.yml                   | 46 ++++++++++++-------
 1 file changed, 29 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
index a9d8da4c402..8bf50e2804e 100644
--- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml
@@ -1,23 +1,29 @@
 ---
 
+- set_fact:
+    matrix_appservice_discord_requires_restart: false
+
 - block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
-    register: matrix_appservice_discord_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_appservice_discord_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_appservice_discord_sqlite_database_path_local }}.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_appservice_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_discord_sqlite_database_path_local }} postgres_connection_string_variable_name=matrix_appservice_discord_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists"
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
+      register: matrix_appservice_discord_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_appservice_discord_sqlite_database_path_local }}"
+              dst: "{{ matrix_appservice_discord_database_connString }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_appservice_discord_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-appservice-discord.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_appservice_discord_requires_restart: true
+      when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists|bool"
   when: "matrix_appservice_discord_database_engine == 'postgres'"
 
 - name: Ensure Appservice Discord image is pulled
@@ -100,3 +106,9 @@
   service:
     daemon_reload: yes
   when: "matrix_appservice_discord_systemd_service_result.changed"
+
+- name: Ensure matrix-appservice-discord.service restarted, if necessary
+  service:
+    name: "matrix-appservice-discord.service"
+    state: restarted
+  when: "matrix_appservice_discord_requires_restart|bool"

From 7248eb3c11d77303dbc395f03c2aeb8ed85e4639 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:25:44 +0200
Subject: [PATCH 50/79] Fix syntax error in
 roles/matrix-bridge-appservice-irc/defaults/main.yml

---
 roles/matrix-bridge-appservice-irc/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml
index a1c762537fb..bf23b0abb05 100644
--- a/roles/matrix-bridge-appservice-irc/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml
@@ -22,7 +22,7 @@ matrix_appservice_irc_database_password: ~
 matrix_appservice_irc_database_hostname: 'matrix-postgres'
 matrix_appservice_irc_database_port: 5432
 matrix_appservice_irc_database_name: matrix_appservice_irc
-matrix_appservice_irc_databasa
+
 matrix_appservice_irc_database_connString: >-2
   {%- if matrix_appservice_irc_database_engine == 'postgres' -%}
   postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable

From 0ca48f3532d211df9e44c67d9661a95420701660 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 16:57:51 +0200
Subject: [PATCH 51/79] Separate matrix-ma1sd install/uninstall tasks

---
 roles/matrix-ma1sd/tasks/main.yml             |  9 ++-
 .../{setup_ma1sd.yml => setup_install.yml}    | 57 ++-----------------
 roles/matrix-ma1sd/tasks/setup_uninstall.yml  | 35 ++++++++++++
 3 files changed, 48 insertions(+), 53 deletions(-)
 rename roles/matrix-ma1sd/tasks/{setup_ma1sd.yml => setup_install.yml} (69%)
 create mode 100644 roles/matrix-ma1sd/tasks/setup_uninstall.yml

diff --git a/roles/matrix-ma1sd/tasks/main.yml b/roles/matrix-ma1sd/tasks/main.yml
index f5ac34d650e..0b8a114e1de 100644
--- a/roles/matrix-ma1sd/tasks/main.yml
+++ b/roles/matrix-ma1sd/tasks/main.yml
@@ -8,7 +8,14 @@
     - setup-all
     - setup-ma1sd
 
-- import_tasks: "{{ role_path }}/tasks/setup_ma1sd.yml"
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_ma1sd_enabled|bool"
+  tags:
+    - setup-all
+    - setup-ma1sd
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_ma1sd_enabled|bool"
   tags:
     - setup-all
     - setup-ma1sd
diff --git a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml b/roles/matrix-ma1sd/tasks/setup_install.yml
similarity index 69%
rename from roles/matrix-ma1sd/tasks/setup_ma1sd.yml
rename to roles/matrix-ma1sd/tasks/setup_install.yml
index 46acb42867e..aa11ae7a31c 100644
--- a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml
+++ b/roles/matrix-ma1sd/tasks/setup_install.yml
@@ -1,9 +1,5 @@
 ---
 
-#
-# Tasks related to setting up ma1sd
-#
-
 - name: Ensure ma1sd paths exist
   file:
     path: "{{ item.path }}"
@@ -15,10 +11,9 @@
     - { path: "{{ matrix_ma1sd_config_path }}", when: true }
     - { path: "{{ matrix_ma1sd_data_path }}", when: true }
     - { path: "{{ matrix_ma1sd_docker_src_files_path }}", when: "{{ matrix_ma1sd_container_image_self_build }}"}
-  when: matrix_ma1sd_enabled|bool and item.when
+  when: "item.when|bool"
 
 - import_tasks: "{{ role_path }}/tasks/migrate_mxisd.yml"
-  when: matrix_ma1sd_enabled|bool
 
 - name: Ensure ma1sd image is pulled
   docker_image:
@@ -26,7 +21,7 @@
     source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
     force_source: "{{ matrix_ma1sd_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_ma1sd_docker_image_force_pull }}"
-  when: matrix_ma1sd_enabled|bool and not matrix_ma1sd_container_image_self_build
+  when: "not matrix_ma1sd_container_image_self_build|bool"
 
 - block:
   - name: Ensure gradle is installed for self-building (Debian)
@@ -72,7 +67,7 @@
       repository: "{{ matrix_ma1sd_docker_image }}"
       force_tag: yes
       source: local
-  when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_container_image_self_build"
+  when: "matrix_ma1sd_container_image_self_build|bool"
 
 - name: Ensure ma1sd config installed
   copy:
@@ -81,7 +76,6 @@
     mode: 0644
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_groupname }}"
-  when: matrix_ma1sd_enabled|bool
 
 - name: Ensure custom templates are installed if any
   copy:
@@ -95,7 +89,7 @@
     - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_session_validation_template }}", location: 'validate-template.eml'}
     - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_unbind_fraudulent_template }}", location: 'unbind-fraudulent.eml'}
     - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_matrixid_template }}", location: 'mxid-template.eml'}
-  when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value"
+  when: "matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value"
 
 - name: Ensure matrix-ma1sd.service installed
   template:
@@ -103,49 +97,8 @@
     dest: "{{ matrix_systemd_path }}/matrix-ma1sd.service"
     mode: 0644
   register: matrix_ma1sd_systemd_service_result
-  when: matrix_ma1sd_enabled|bool
 
 - name: Ensure systemd reloaded after matrix-ma1sd.service installation
   service:
     daemon_reload: yes
-  when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_systemd_service_result.changed"
-
-#
-# Tasks related to getting rid of ma1sd (if it was previously enabled)
-#
-
-- name: Check existence of matrix-ma1sd service
-  stat:
-    path: "{{ matrix_systemd_path }}/matrix-ma1sd.service"
-  register: matrix_ma1sd_service_stat
-
-- name: Ensure matrix-ma1sd is stopped
-  service:
-    name: matrix-ma1sd
-    state: stopped
-    daemon_reload: yes
-  register: stopping_result
-  when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists"
-
-- name: Ensure matrix-ma1sd.service doesn't exist
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-ma1sd.service"
-    state: absent
-  when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists"
-
-- name: Ensure systemd reloaded after matrix-ma1sd.service removal
-  service:
-    daemon_reload: yes
-  when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists"
-
-- name: Ensure Matrix ma1sd paths don't exist
-  file:
-    path: "{{ matrix_ma1sd_base_path }}"
-    state: absent
-  when: "not matrix_ma1sd_enabled|bool"
-
-- name: Ensure ma1sd Docker image doesn't exist
-  docker_image:
-    name: "{{ matrix_ma1sd_docker_image }}"
-    state: absent
-  when: "not matrix_ma1sd_enabled|bool"
+  when: "matrix_ma1sd_systemd_service_result.changed|bool"
diff --git a/roles/matrix-ma1sd/tasks/setup_uninstall.yml b/roles/matrix-ma1sd/tasks/setup_uninstall.yml
new file mode 100644
index 00000000000..b36ab508f10
--- /dev/null
+++ b/roles/matrix-ma1sd/tasks/setup_uninstall.yml
@@ -0,0 +1,35 @@
+---
+
+- name: Check existence of matrix-ma1sd service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-ma1sd.service"
+  register: matrix_ma1sd_service_stat
+
+- name: Ensure matrix-ma1sd is stopped
+  service:
+    name: matrix-ma1sd
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "matrix_ma1sd_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-ma1sd.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-ma1sd.service"
+    state: absent
+  when: "matrix_ma1sd_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-ma1sd.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_ma1sd_service_stat.stat.exists|bool"
+
+- name: Ensure Matrix ma1sd paths don't exist
+  file:
+    path: "{{ matrix_ma1sd_base_path }}"
+    state: absent
+
+- name: Ensure ma1sd Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_ma1sd_docker_image }}"
+    state: absent

From af3ea67bbaa699ad6f03db6633f1dd1902ab9290 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 17:16:25 +0200
Subject: [PATCH 52/79] Add (SQLite + Postgres) support and automatic migration
 to matrix-ma1sd

---
 group_vars/matrix_servers                     | 10 ++++++
 roles/matrix-ma1sd/defaults/main.yml          | 22 ++++++++++++
 roles/matrix-ma1sd/tasks/setup_install.yml    | 36 +++++++++++++++++++
 roles/matrix-ma1sd/templates/ma1sd.yaml.j2    | 16 +++++++--
 .../tasks/util/migrate_db_to_postgres.yml     |  2 +-
 5 files changed, 82 insertions(+), 4 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index d6d6cc4a07b..735d132439b 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -835,6 +835,10 @@ matrix_ma1sd_systemd_wanted_services_list: |
     (['matrix-mailer.service'] if matrix_mailer_enabled else [])
   }}
 
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_ma1sd_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_ma1sd_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ma1sd.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-ma1sd
@@ -953,6 +957,12 @@ matrix_postgres_db_name: "homeserver"
 
 matrix_postgres_additional_databases: |
   {{
+    ([{
+      'name': matrix_ma1sd_database_db_name,
+      'username': matrix_ma1sd_database_username,
+      'password': matrix_ma1sd_database_password,
+    }] if (matrix_ma1sd_enabled and matrix_ma1sd_database_engine == 'postgres' and matrix_ma1sd_database_hostname == 'matrix-postgres') else [])
+    +
     ([{
       'name': matrix_bot_matrix_reminder_bot_database_db_name,
       'username': matrix_bot_matrix_reminder_bot_database_username,
diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml
index f7a0782d5fb..42e36e924fe 100644
--- a/roles/matrix-ma1sd/defaults/main.yml
+++ b/roles/matrix-ma1sd/defaults/main.yml
@@ -39,6 +39,28 @@ matrix_ma1sd_systemd_wanted_services_list: []
 # Enabling this is discouraged. Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/features/identity.md#lookups
 matrix_ma1sd_matrixorg_forwarding_enabled: false
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_ma1sd_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_ma1sd_postgres_*` variables
+matrix_ma1sd_database_engine: 'sqlite'
+
+matrix_ma1sd_sqlite_database_path_local: "{{ matrix_ma1sd_data_path }}/ma1sd.db"
+matrix_ma1sd_sqlite_database_path_in_container: "/var/ma1sd/ma1sd.db"
+
+matrix_ma1sd_database_username: 'matrix_ma1sd'
+matrix_ma1sd_database_password: 'some-password'
+matrix_ma1sd_database_hostname: 'matrix-postgres'
+matrix_ma1sd_database_port: 5432
+matrix_ma1sd_database_db_name: 'matrix_ma1sd'
+
+matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}'
+
+
 # ma1sd has serveral supported identity stores.
 # One of them is storing identities directly in Synapse's database.
 # Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/stores/synapse.md
diff --git a/roles/matrix-ma1sd/tasks/setup_install.yml b/roles/matrix-ma1sd/tasks/setup_install.yml
index aa11ae7a31c..9ae5f07798f 100644
--- a/roles/matrix-ma1sd/tasks/setup_install.yml
+++ b/roles/matrix-ma1sd/tasks/setup_install.yml
@@ -15,6 +15,36 @@
 
 - import_tasks: "{{ role_path }}/tasks/migrate_mxisd.yml"
 
+
+# These (SQLite -> Postgres) migration tasks are usually at the top,
+# but we'd like to run them after `migrate_mxisd.yml`, which requires the ma1sd paths to exist.
+- set_fact:
+    matrix_ma1sd_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_ma1sd_sqlite_database_path_local }}"
+      register: matrix_ma1sd_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_ma1sd_sqlite_database_path_local }}"
+              dst: "{{ matrix_ma1sd_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_ma1sd_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-ma1sd.service']
+              pgloader_options: ['--with "quote identifiers"']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_ma1sd_requires_restart: true
+      when: "matrix_ma1sd_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_ma1sd_database_engine == 'postgres'"
+
 - name: Ensure ma1sd image is pulled
   docker_image:
     name: "{{ matrix_ma1sd_docker_image }}"
@@ -102,3 +132,9 @@
   service:
     daemon_reload: yes
   when: "matrix_ma1sd_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-ma1sd.service restarted, if necessary
+  service:
+    name: "matrix-ma1sd.service"
+    state: restarted
+  when: "matrix_ma1sd_requires_restart|bool"
diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
index 84585707413..43b4022dca9 100644
--- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
+++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
@@ -11,9 +11,19 @@ key:
   path: /var/ma1sd/sign.key
 
 storage:
-  provider:
-    sqlite:
-      database: /var/ma1sd/ma1sd.db
+  {% if matrix_ma1sd_database_engine == 'sqlite' %}
+    backend: sqlite
+    provider:
+      sqlite:
+        database: {{ matrix_ma1sd_sqlite_database_path_in_container|to_json }}
+  {% elif matrix_ma1sd_database_engine == 'postgres' %}
+    backend: postgresql
+    provider:
+      postgresql:
+        database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}
+        username: {{ matrix_ma1sd_database_username|to_json }}
+        password: {{ matrix_ma1sd_database_password|to_json }}
+  {% endif %}
 
 {% if matrix_ma1sd_dns_overwrite_enabled %}
 dns:
diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index b77ce4b00c3..5d917538e3d 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -93,7 +93,7 @@
       --entrypoint=/bin/sh
       {{ matrix_postgres_pgloader_docker_image }}
       -c
-      'pgloader /in.db {{ matrix_postgres_db_migration_request.dst }}'
+      'pgloader {{ matrix_postgres_db_migration_request.pgloader_options|default([])|join(' ') }} /in.db {{ matrix_postgres_db_migration_request.dst }}'
 
 - name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup)
   command:

From 3ab7dd1abe07574085da2705b3f79ca6773c4f29 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 17:22:31 +0200
Subject: [PATCH 53/79] Potentially add matrix-postgres.service as required for
 bridges, etc.

---
 group_vars/matrix_servers | 41 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 735d132439b..1223d8a62c5 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -61,6 +61,8 @@ matrix_appservice_discord_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_appservice_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'discord.as.token') | to_uuid }}"
@@ -207,6 +209,8 @@ matrix_mautrix_facebook_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mautrix_facebook_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'fb.as.token') | to_uuid }}"
@@ -244,6 +248,8 @@ matrix_mautrix_hangouts_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mautrix_hangouts_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ho.as.token') | to_uuid }}"
@@ -282,6 +288,8 @@ matrix_mautrix_telegram_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mautrix_telegram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'telegr.as.token') | to_uuid }}"
@@ -318,6 +326,8 @@ matrix_mautrix_whatsapp_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mautrix_whatsapp_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'whats.as.token') | to_uuid }}"
@@ -378,6 +388,8 @@ matrix_mx_puppet_skype_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_skype_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'skype.as.tok') | to_uuid }}"
@@ -413,6 +425,8 @@ matrix_mx_puppet_slack_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_slack_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxslk.as.tok') | to_uuid }}"
@@ -447,6 +461,8 @@ matrix_mx_puppet_twitter_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_twitter_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxtwt.as.tok') | to_uuid }}"
@@ -484,6 +500,8 @@ matrix_mx_puppet_instagram_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_instagram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxig.as.tok') | to_uuid }}"
@@ -518,6 +536,8 @@ matrix_mx_puppet_discord_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxdsc.as.tok') | to_uuid }}"
@@ -552,6 +572,8 @@ matrix_mx_puppet_steam_systemd_required_services_list: |
     ['docker.service']
     +
     (['matrix-synapse.service'] if matrix_synapse_enabled else [])
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
 matrix_mx_puppet_steam_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxste.as.tok') | to_uuid }}"
@@ -580,6 +602,13 @@ matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key
 # We don't enable bots by default.
 matrix_bot_matrix_reminder_bot_enabled: false
 
+matrix_bot_matrix_reminder_bot_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+  }}
+
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_bot_matrix_reminder_bot_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
 matrix_bot_matrix_reminder_bot_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'reminder.bot.db') | to_uuid }}"
@@ -678,6 +707,13 @@ matrix_dimension_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_ena
 matrix_integration_manager_rest_url: "{{ matrix_dimension_integrations_rest_url if matrix_dimension_enabled else None }}"
 matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if matrix_dimension_enabled else None }}"
 
+matrix_dimension_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+  }}
+
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
 matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
@@ -826,6 +862,11 @@ matrix_ma1sd_threepid_medium_email_connectors_smtp_tls: 0
 
 matrix_ma1sd_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}"
 
+matrix_ma1sd_systemd_required_services_list: |
+  {{
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+  }}
+
 matrix_ma1sd_systemd_wanted_services_list: |
   {{
     (['matrix-corporal.service'] if matrix_corporal_enabled else ['matrix-synapse.service'])

From 13d8a9b39c1a5c5a225ad2c82d1a876210e341c0 Mon Sep 17 00:00:00 2001
From: transcaffeine <transcaffeine@finallycoffee.eu>
Date: Mon, 14 Dec 2020 16:27:06 +0100
Subject: [PATCH 54/79] hint supported automatic migration nedb->postgres

---
 .../tasks/validate_config.yml                 |   2 +-
 roles/matrix-postgres/tasks/import_nedb.yml   | 117 ++++++++++++++++++
 roles/matrix-postgres/tasks/main.yml          |  16 +++
 3 files changed, 134 insertions(+), 1 deletion(-)
 create mode 100644 roles/matrix-postgres/tasks/import_nedb.yml

diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
index e0e932bcc14..9b89a3400f6 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
@@ -49,7 +49,7 @@
         To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command.
         Alternatively, to migrate your existing neDB database to Postgres:
         1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`)
+        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-irc-nedb --extra-vars='nedb_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`)
         3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
     when: "matrix_appservice_irc_nedb_stat_result.stat.exists"
   when: "matrix_appservice_irc_database_engine == 'postgres'"
diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml
new file mode 100644
index 00000000000..cc1f9d78815
--- /dev/null
+++ b/roles/matrix-postgres/tasks/import_nedb.yml
@@ -0,0 +1,117 @@
+---
+
+# Pre-checks
+
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import."
+  when: "not matrix_postgres_enabled|bool"
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars"
+  when: "nedb_database_path is not defined or nedb_database_path.startswith('<')"
+
+- name: Check if the provided nedb database file exists
+  stat:
+    path: "{{ nedb_database_path }}"
+  register: nedb_database_path_stat_result
+
+- name: Fail if provided SQLite database file doesn't exist
+  fail:
+    msg: "File cannot be found on the server at {{ nedb_database_path }}"
+  when: "not nedb_database_path_stat_result.stat.exists"
+
+# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string,
+# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string.
+
+- block:
+    - name: Fail if postgres_connection_string_variable_name points to an undefined variable
+      fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`"
+      when: "postgres_connection_string_variable_name not in vars"
+
+    - name: Get Postgres connection string from variable
+      set_fact:
+        postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}"
+  when: 'postgres_connection_string_variable_name is defined'
+
+- name: Fail if playbook called incorrectly
+  fail:
+    msg: >-
+      Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`.
+      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"`
+  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
+
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+# See https://github.com/matrix-org/matrix-appservice-irc/wiki/Migrating-from-NEdB-to-PostgreSQL
+- name: Import appservice_irc NeDB database from {{ sqlite_database_path }} into Postgres
+  when: database == 'appservice_irc'
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_irc_docker_image }}
+      -c
+      './scripts/migrate-db-to-pgres.sh -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
+
+# No migration.sh available, but found this:
+# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts
+# Usage should be similar to appservice_irc
+- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres
+  when: database == 'appservice_slack'
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_slack_docker_image }}
+      -c
+      'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
+
+- name: Archive NeDB database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup)
+  command:
+    cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup"
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
+        ]
+      }}
diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml
index b9c2ae7c99c..5180131478d 100644
--- a/roles/matrix-postgres/tasks/main.yml
+++ b/roles/matrix-postgres/tasks/main.yml
@@ -32,6 +32,22 @@
   tags:
     - import-generic-sqlite-db
 
+# Imports appservice-irc NeDB into postgres
+- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
+  vars:
+    database: appservice_irc
+  when: run_postgres_import_nedb|bool
+  tags:
+    - import-irc-nedb
+
+# Imports slacks neDB to postgres.
+- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
+  vars:
+    database: appservice_slack
+  when: run_postgres_import_nedb|bool
+  tags:
+    - import-slack-nedb
+
 - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
   when: run_postgres_upgrade|bool
   tags:

From 516ccb2b2b3eee4d110a6056302d80dcf902f03a Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 18:12:03 +0200
Subject: [PATCH 55/79] Separate matrix-registration install/uninstall tasks

---
 roles/matrix-registration/tasks/main.yml      |  9 +++-
 .../tasks/{setup.yml => setup_install.yml}    | 50 ++-----------------
 .../tasks/setup_uninstall.yml                 | 30 +++++++++++
 3 files changed, 43 insertions(+), 46 deletions(-)
 rename roles/matrix-registration/tasks/{setup.yml => setup_install.yml} (57%)
 create mode 100644 roles/matrix-registration/tasks/setup_uninstall.yml

diff --git a/roles/matrix-registration/tasks/main.yml b/roles/matrix-registration/tasks/main.yml
index 4a884ccd9c2..4fef6abe97d 100644
--- a/roles/matrix-registration/tasks/main.yml
+++ b/roles/matrix-registration/tasks/main.yml
@@ -8,7 +8,14 @@
     - setup-all
     - setup-matrix-registration
 
-- import_tasks: "{{ role_path }}/tasks/setup.yml"
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: "run_setup|bool and matrix_registration_enabled|bool"
+  tags:
+    - setup-all
+    - setup-matrix-registration
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: "run_setup|bool and not matrix_registration_enabled|bool"
   tags:
     - setup-all
     - setup-matrix-registration
diff --git a/roles/matrix-registration/tasks/setup.yml b/roles/matrix-registration/tasks/setup_install.yml
similarity index 57%
rename from roles/matrix-registration/tasks/setup.yml
rename to roles/matrix-registration/tasks/setup_install.yml
index dfe1ba8ba6c..77c35581ff6 100644
--- a/roles/matrix-registration/tasks/setup.yml
+++ b/roles/matrix-registration/tasks/setup_install.yml
@@ -1,9 +1,5 @@
 ---
 
-#
-# Tasks related to setting up matrix-registration
-#
-
 - name: Ensure matrix-registration paths exist
   file:
     path: "{{ item.path }}"
@@ -16,7 +12,7 @@
     - { path: "{{ matrix_registration_config_path }}", when: true }
     - { path: "{{ matrix_registration_data_path }}", when: true }
     - { path: "{{ matrix_registration_docker_src_files_path }}", when: "{{ matrix_registration_container_image_self_build }}"}
-  when: matrix_registration_enabled|bool and item.when
+  when: "item.when|bool"
 
 - name: Ensure matrix-registration image is pulled
   docker_image:
@@ -24,7 +20,7 @@
     source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
     force_source: "{{ matrix_registration_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_registration_docker_image_force_pull }}"
-  when: "matrix_registration_enabled|bool and not matrix_registration_container_image_self_build|bool"
+  when: "not matrix_registration_container_image_self_build|bool"
 
 - name: Ensure matrix-registration repository is present when self-building
   git:
@@ -33,7 +29,7 @@
     version: "{{ matrix_registration_container_image_self_build_branch }}"
     force: "yes"
   register: matrix_registration_git_pull_results
-  when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool"
+  when: "matrix_registration_container_image_self_build|bool"
 
 - name: Ensure matrix-registration Docker image is built
   docker_image:
@@ -44,7 +40,7 @@
       dockerfile: Dockerfile
       path: "{{ matrix_registration_docker_src_files_path }}"
       pull: yes
-  when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool"
+  when: "matrix_registration_container_image_self_build|bool"
 
 - name: Ensure matrix-registration config installed
   copy:
@@ -53,7 +49,6 @@
     mode: 0644
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_groupname }}"
-  when: matrix_registration_enabled|bool
 
 - name: Ensure matrix-registration.service installed
   template:
@@ -61,43 +56,8 @@
     dest: "{{ matrix_systemd_path }}/matrix-registration.service"
     mode: 0644
   register: matrix_registration_systemd_service_result
-  when: matrix_registration_enabled|bool
 
 - name: Ensure systemd reloaded after matrix-registration.service installation
   service:
     daemon_reload: yes
-  when: "matrix_registration_enabled|bool and matrix_registration_systemd_service_result.changed"
-
-#
-# Tasks related to getting rid of matrix-registration (if it was previously enabled)
-#
-
-- name: Check existence of matrix-registration service
-  stat:
-    path: "{{ matrix_systemd_path }}/matrix-registration.service"
-  register: matrix_registration_service_stat
-
-- name: Ensure matrix-registration is stopped
-  service:
-    name: matrix-registration
-    state: stopped
-    daemon_reload: yes
-  register: stopping_result
-  when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists"
-
-- name: Ensure matrix-registration.service doesn't exist
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-registration.service"
-    state: absent
-  when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists"
-
-- name: Ensure systemd reloaded after matrix-registration.service removal
-  service:
-    daemon_reload: yes
-  when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists"
-
-- name: Ensure matrix-registration Docker image doesn't exist
-  docker_image:
-    name: "{{ matrix_registration_docker_image }}"
-    state: absent
-  when: "not matrix_registration_enabled|bool"
+  when: "matrix_registration_systemd_service_result.changed|bool"
diff --git a/roles/matrix-registration/tasks/setup_uninstall.yml b/roles/matrix-registration/tasks/setup_uninstall.yml
new file mode 100644
index 00000000000..573f8170b41
--- /dev/null
+++ b/roles/matrix-registration/tasks/setup_uninstall.yml
@@ -0,0 +1,30 @@
+---
+
+- name: Check existence of matrix-registration service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-registration.service"
+  register: matrix_registration_service_stat
+
+- name: Ensure matrix-registration is stopped
+  service:
+    name: matrix-registration
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "matrix_registration_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-registration.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-registration.service"
+    state: absent
+  when: "matrix_registration_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-registration.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_registration_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-registration Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_registration_docker_image }}"
+    state: absent

From 8d7459387895a6a3c14ef4c1a751353c740c5a9d Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 18:58:37 +0200
Subject: [PATCH 56/79] Prepare matrix-registation for (SQLite + Postgres)
 support

Auto-migration and everything seems to work. It's just that
matrix-registration cannot load the Python modules required
for talking to a Postgres database.

Tracked here: https://github.com/ZerataX/matrix-registration/issues/44

Until this gets fixed, we'll continue default to 'sqlite'.
---
 group_vars/matrix_servers                     | 18 +++++++++++
 roles/matrix-registration/defaults/main.yml   | 30 +++++++++++++++++
 .../tasks/setup_install.yml                   | 32 +++++++++++++++++++
 .../tasks/validate_config.yml                 |  7 ++++
 .../templates/config.yaml.j2                  |  2 +-
 5 files changed, 88 insertions(+), 1 deletion(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 1223d8a62c5..ad62a86edbd 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1010,6 +1010,12 @@ matrix_postgres_additional_databases: |
       'password': matrix_bot_matrix_reminder_bot_database_password,
     }] if (matrix_bot_matrix_reminder_bot_enabled and matrix_bot_matrix_reminder_bot_database_engine == 'postgres' and matrix_bot_matrix_reminder_bot_database_hostname == 'matrix-postgres') else [])
     +
+    ([{
+      'name': matrix_registration_database_db_name,
+      'username': matrix_registration_database_username,
+      'password': matrix_registration_database_password,
+    }] if (matrix_registration_enabled and matrix_registration_database_engine == 'postgres' and matrix_registration_database_hostname == 'matrix-postgres') else [])
+    +
     ([{
       'name': matrix_appservice_discord_database_db_name,
       'username': matrix_appservice_discord_database_username,
@@ -1291,6 +1297,18 @@ matrix_registration_api_validate_certs: "{{ false if matrix_ssl_retrieval_method
 
 matrix_registration_container_image_self_build: "{{ matrix_architecture != 'amd64' }}"
 
+matrix_registration_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+  }}
+
+# We'd like to use 'postgres' if matrix_postgres_enabled, but the container image doesn't seem to support that.
+# Learn more here: https://github.com/ZerataX/matrix-registration/issues/44
+matrix_registration_database_engine: 'sqlite'
+matrix_registration_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx.registr.db') | to_uuid }}"
+
 ######################################################################
 #
 # /matrix-registration
diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml
index 95147665256..5afe4dee350 100644
--- a/roles/matrix-registration/defaults/main.yml
+++ b/roles/matrix-registration/defaults/main.yml
@@ -32,6 +32,36 @@ matrix_registration_systemd_wanted_services_list: []
 # Takes an "<ip>:<port>" or "<port>" value (e.g. "127.0.0.1:8767"), or empty string to not expose.
 matrix_registration_container_http_host_bind_port: ''
 
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_registration_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_registration_postgres_*` variables
+matrix_registration_database_engine: 'sqlite'
+
+matrix_registration_sqlite_database_path_local: "{{ matrix_registration_data_path }}/db.sqlite3"
+matrix_registration_sqlite_database_path_in_container: "/data/db.sqlite3"
+
+matrix_registration_database_username: 'matrix_registration'
+matrix_registration_database_password: 'some-password'
+matrix_registration_database_hostname: 'matrix-postgres'
+matrix_registration_database_port: 5432
+matrix_registration_database_db_name: 'matrix_registration'
+
+matrix_registration_database_connection_string: 'postgresql://{{ matrix_registration_database_username }}:{{ matrix_registration_database_password }}@{{ matrix_registration_database_hostname }}:{{ matrix_registration_database_port }}/{{ matrix_registration_database_db_name }}'
+
+# For some reason, matrix-registraiton expects the `db` field to be like this: `sqlite:////data/db.sqlite3`.
+# (seems like one too many slashes, but..)
+matrix_registration_db: "{{
+	{
+		'sqlite': ('sqlite:///' + matrix_registration_sqlite_database_path_in_container),
+		'postgres': matrix_registration_database_connection_string,
+	}[matrix_registration_database_engine]
+}}"
+
+
 # The path at which Matrix Registration will be exposed on `matrix.DOMAIN`
 # (only applies when matrix-nginx-proxy is used).
 matrix_registration_public_endpoint: /matrix-registration
diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml
index 77c35581ff6..489bab8ba80 100644
--- a/roles/matrix-registration/tasks/setup_install.yml
+++ b/roles/matrix-registration/tasks/setup_install.yml
@@ -1,5 +1,31 @@
 ---
 
+- set_fact:
+    matrix_registration_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_registration_sqlite_database_path_local }}"
+      register: matrix_registration_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_registration_sqlite_database_path_local }}"
+              dst: "{{ matrix_registration_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_registration_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-registration.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_registration_requires_restart: true
+      when: "matrix_registration_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_registration_database_engine == 'postgres'"
+
 - name: Ensure matrix-registration paths exist
   file:
     path: "{{ item.path }}"
@@ -61,3 +87,9 @@
   service:
     daemon_reload: yes
   when: "matrix_registration_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-registration.service restarted, if necessary
+  service:
+    name: "matrix-registration.service"
+    state: restarted
+  when: "matrix_registration_requires_restart|bool"
diff --git a/roles/matrix-registration/tasks/validate_config.yml b/roles/matrix-registration/tasks/validate_config.yml
index 90466b46c48..80293bcba3a 100644
--- a/roles/matrix-registration/tasks/validate_config.yml
+++ b/roles/matrix-registration/tasks/validate_config.yml
@@ -18,3 +18,10 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_registration_docker_repo', 'new': 'matrix_registration_container_image_self_build_repo'}
+
+- name: Fail if Postgres usage attempted
+  fail:
+    msg: >
+      matrix-registration doesn't support using Postgres just yet.
+      Learn more here: https://github.com/ZerataX/matrix-registration/issues/44
+  when: "matrix_registration_database_engine == 'postgres'"
diff --git a/roles/matrix-registration/templates/config.yaml.j2 b/roles/matrix-registration/templates/config.yaml.j2
index f3b1c57b72d..ae0ba5ff226 100644
--- a/roles/matrix-registration/templates/config.yaml.j2
+++ b/roles/matrix-registration/templates/config.yaml.j2
@@ -3,7 +3,7 @@ server_name: {{ matrix_registration_server_name|to_json }}
 shared_secret: {{ matrix_registration_shared_secret|to_json }}
 admin_secret: {{ matrix_registration_admin_secret|to_json }}
 riot_instance: {{ matrix_registration_riot_instance|to_json }}
-db: 'sqlite:////data/db.sqlite3'
+db: {{ matrix_registration_db|to_json }}
 host: '0.0.0.0'
 port: 5000
 rate_limit: ["100 per day", "10 per minute"]

From 374f43735aa362bbbb2a9068ef2fd8e81b5ab561 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 20:05:31 +0200
Subject: [PATCH 57/79] Separate matrix-dimension install/uninstall tasks

---
 roles/matrix-dimension/tasks/main.yml         | 10 ++-
 .../tasks/setup_dimension.yml                 | 85 -------------------
 .../matrix-dimension/tasks/setup_install.yml  | 36 ++++++++
 .../tasks/setup_uninstall.yml                 | 35 ++++++++
 4 files changed, 79 insertions(+), 87 deletions(-)
 delete mode 100644 roles/matrix-dimension/tasks/setup_dimension.yml
 create mode 100644 roles/matrix-dimension/tasks/setup_install.yml
 create mode 100644 roles/matrix-dimension/tasks/setup_uninstall.yml

diff --git a/roles/matrix-dimension/tasks/main.yml b/roles/matrix-dimension/tasks/main.yml
index 1888f9452f1..aad552866a2 100644
--- a/roles/matrix-dimension/tasks/main.yml
+++ b/roles/matrix-dimension/tasks/main.yml
@@ -8,8 +8,14 @@
     - setup-all
     - setup-dimension
 
-- import_tasks: "{{ role_path }}/tasks/setup_dimension.yml"
-  when: run_setup|bool
+- import_tasks: "{{ role_path }}/tasks/setup_install.yml"
+  when: run_setup|bool and matrix_dimension_enabled|bool
+  tags:
+    - setup-all
+    - setup-dimension
+
+- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml"
+  when: run_setup|bool and not matrix_dimension_enabled|bool
   tags:
     - setup-all
     - setup-dimension
diff --git a/roles/matrix-dimension/tasks/setup_dimension.yml b/roles/matrix-dimension/tasks/setup_dimension.yml
deleted file mode 100644
index 2437a547223..00000000000
--- a/roles/matrix-dimension/tasks/setup_dimension.yml
+++ /dev/null
@@ -1,85 +0,0 @@
----
-
-#
-# Tasks related to setting up the dimension
-#
-
-- name: Ensure Dimension base path exists
-  file:
-    path: "{{ matrix_dimension_base_path }}"
-    state: directory
-    mode: 0770
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_dimension_user_gid }}"
-  when: matrix_dimension_enabled|bool
-
-- name: Ensure Dimension config installed
-  copy:
-    content: "{{ matrix_dimension_configuration|to_nice_yaml }}"
-    dest: "{{ matrix_dimension_base_path }}/config.yaml"
-    mode: 0640
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_dimension_user_gid }}"
-  when: matrix_dimension_enabled|bool
-
-- name: Ensure Dimension image is pulled
-  docker_image:
-    name: "{{ matrix_dimension_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_dimension_docker_image_force_pull }}"
-  when: matrix_dimension_enabled|bool
-
-- name: Ensure matrix-dimension.service installed
-  template:
-    src: "{{ role_path }}/templates/systemd/matrix-dimension.service.j2"
-    dest: "{{ matrix_systemd_path }}/matrix-dimension.service"
-    mode: 0644
-  register: matrix_dimension_systemd_service_result
-  when: matrix_dimension_enabled|bool
-
-- name: Ensure systemd reloaded after matrix-dimension.service installation
-  service:
-    daemon_reload: yes
-  when: "matrix_dimension_enabled|bool and matrix_dimension_systemd_service_result.changed"
-
-#
-# Tasks related to getting rid of the dimension (if it was previously enabled)
-#
-
-- name: Check existence of matrix-dimension service
-  stat:
-    path: "{{ matrix_systemd_path }}/matrix-dimension.service"
-  register: matrix_dimension_service_stat
-  when: "not matrix_dimension_enabled|bool"
-
-- name: Ensure matrix-dimension is stopped
-  service:
-    name: matrix-dimension
-    state: stopped
-    daemon_reload: yes
-  register: stopping_result
-  when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists"
-
-- name: Ensure matrix-dimension.service doesn't exist
-  file:
-    path: "{{ matrix_systemd_path }}/matrix-dimension.service"
-    state: absent
-  when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists"
-
-- name: Ensure systemd reloaded after matrix-dimension.service removal
-  service:
-    daemon_reload: yes
-  when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists"
-
-- name: Ensure Dimension environment variables path doesn't exist
-  file:
-    path: "{{ matrix_dimension_base_path }}"
-    state: absent
-  when: "not matrix_dimension_enabled|bool"
-
-- name: Ensure Dimension Docker image doesn't exist
-  docker_image:
-    name: "{{ matrix_dimension_docker_image }}"
-    state: absent
-  when: "not matrix_dimension_enabled|bool"
diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml
new file mode 100644
index 00000000000..e118dd62304
--- /dev/null
+++ b/roles/matrix-dimension/tasks/setup_install.yml
@@ -0,0 +1,36 @@
+---
+
+- name: Ensure Dimension base path exists
+  file:
+    path: "{{ matrix_dimension_base_path }}"
+    state: directory
+    mode: 0770
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_dimension_user_gid }}"
+
+- name: Ensure Dimension config installed
+  copy:
+    content: "{{ matrix_dimension_configuration|to_nice_yaml }}"
+    dest: "{{ matrix_dimension_base_path }}/config.yaml"
+    mode: 0640
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_dimension_user_gid }}"
+
+- name: Ensure Dimension image is pulled
+  docker_image:
+    name: "{{ matrix_dimension_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_dimension_docker_image_force_pull }}"
+
+- name: Ensure matrix-dimension.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-dimension.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-dimension.service"
+    mode: 0644
+  register: matrix_dimension_systemd_service_result
+
+- name: Ensure systemd reloaded after matrix-dimension.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_dimension_systemd_service_result.changed|bool"
diff --git a/roles/matrix-dimension/tasks/setup_uninstall.yml b/roles/matrix-dimension/tasks/setup_uninstall.yml
new file mode 100644
index 00000000000..9bc4ac8b9e5
--- /dev/null
+++ b/roles/matrix-dimension/tasks/setup_uninstall.yml
@@ -0,0 +1,35 @@
+---
+
+- name: Check existence of matrix-dimension service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-dimension.service"
+  register: matrix_dimension_service_stat
+
+- name: Ensure matrix-dimension is stopped
+  service:
+    name: matrix-dimension
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "matrix_dimension_service_stat.stat.exists|bool"
+
+- name: Ensure matrix-dimension.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-dimension.service"
+    state: absent
+  when: "matrix_dimension_service_stat.stat.exists|bool"
+
+- name: Ensure systemd reloaded after matrix-dimension.service removal
+  service:
+    daemon_reload: yes
+  when: "matrix_dimension_service_stat.stat.exists|bool"
+
+- name: Ensure Dimension base directory doesn't exist
+  file:
+    path: "{{ matrix_dimension_base_path }}"
+    state: absent
+
+- name: Ensure Dimension Docker image doesn't exist
+  docker_image:
+    name: "{{ matrix_dimension_docker_image }}"
+    state: absent

From 0790a7b2a8bc21e34d12f4436b88e9a3ce75c526 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 20:31:07 +0200
Subject: [PATCH 58/79] Add support for
 matrix_dimension_systemd_{required,wanted}_services_list

We were referencing them from `group_vars/matrix_servers` since
recently, but there were no such variables and they weren't being put to
use.
---
 roles/matrix-dimension/defaults/main.yml                 | 6 ++++++
 .../templates/systemd/matrix-dimension.service.j2        | 9 +++++++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml
index 3b69227abd1..7ead6e89a45 100644
--- a/roles/matrix-dimension/defaults/main.yml
+++ b/roles/matrix-dimension/defaults/main.yml
@@ -15,6 +15,12 @@ matrix_dimension_base_path: "{{ matrix_base_data_path }}/dimension"
 matrix_dimension_docker_image: "docker.io/turt2live/matrix-dimension:latest"
 matrix_dimension_docker_image_force_pull: "{{ matrix_dimension_docker_image.endswith(':latest') }}"
 
+# List of systemd services that matrix-dimension.service depends on.
+matrix_dimension_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-dimension.service wants
+matrix_dimension_systemd_wanted_services_list: []
+
 # The user and group id correspond to the node user in the `turt2live/matrix-dimension` image.
 matrix_dimension_user_uid: '1000'
 matrix_dimension_user_gid: '1000'
diff --git a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
index 94c38491059..1c900a162f4 100644
--- a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
+++ b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
@@ -1,8 +1,13 @@
 #jinja2: lstrip_blocks: "True"
 [Unit]
 Description=Matrix Dimension
-After=docker.service
-Requires=docker.service
+{% for service in matrix_dimension_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_dimension_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
 DefaultDependencies=no
 
 [Service]

From 2a502db2394af179bcb6def67ac89c7fd83d2d95 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 21:01:47 +0200
Subject: [PATCH 59/79] Add (SQLite + Postgres) support and automatic migration
 to matrix-dimension

---
 group_vars/matrix_servers                     |  9 +++--
 roles/matrix-dimension/defaults/main.yml      | 22 +++++++++++++
 .../matrix-dimension/tasks/setup_install.yml  | 33 +++++++++++++++++++
 .../matrix-dimension/templates/config.yaml.j2 |  6 +++-
 .../systemd/matrix-dimension.service.j2       |  4 ++-
 5 files changed, 67 insertions(+), 7 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index ad62a86edbd..d7163ab7626 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -716,7 +716,7 @@ matrix_dimension_systemd_required_services_list: |
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
 matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
-matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.dimension.db') | to_uuid }}"
+matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db') | to_uuid }}"
 
 ######################################################################
 #
@@ -1092,13 +1092,12 @@ matrix_postgres_additional_databases: |
     }] if (matrix_mx_puppet_steam_enabled
            and matrix_mx_puppet_steam_database_engine == 'postgres'
            and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
-     + ([{
+     +
+    ([{
       'name': matrix_dimension_database_db_name,
       'username': matrix_dimension_database_username,
       'password': matrix_dimension_database_password,
-    }] if (matrix_dimension_enabled
-           and matrix_dimension_database_engine == 'postgres'
-           and matrix_dimension_database_hostname == 'matrix-postgres') else [])
+    }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else [])
    }}
 
 ######################################################################
diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml
index 7ead6e89a45..a6a3bf5d40d 100644
--- a/roles/matrix-dimension/defaults/main.yml
+++ b/roles/matrix-dimension/defaults/main.yml
@@ -40,6 +40,28 @@ matrix_dimension_integrations_jitsi_widget_url: "https://{{ matrix_server_fqn_di
 
 matrix_dimension_homeserver_federationUrl: "http://matrix-synapse:8048"
 
+
+# Database-related configuration fields.
+#
+# To use SQLite, stick to these defaults.
+#
+# To use Postgres:
+# - change the engine (`matrix_dimension_database_engine: 'postgres'`)
+# - adjust your database credentials via the `matrix_dimension_postgres_*` variables
+matrix_dimension_database_engine: 'sqlite'
+
+matrix_dimension_sqlite_database_path_local: "{{ matrix_dimension_base_path }}/dimension.db"
+matrix_dimension_sqlite_database_path_in_container: "dimension.db"
+
+matrix_dimension_database_username: 'matrix_dimension'
+matrix_dimension_database_password: 'some-password'
+matrix_dimension_database_hostname: 'matrix-postgres'
+matrix_dimension_database_port: 5432
+matrix_dimension_database_db_name: 'matrix_dimension'
+
+matrix_dimension_database_connection_string: 'postgres://{{ matrix_dimension_database_username }}:{{ matrix_dimension_database_password }}@{{ matrix_dimension_database_hostname }}:{{ matrix_dimension_database_port }}/{{ matrix_dimension_database_db_name }}'
+
+
 # Default Dimension configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml
index e118dd62304..9a264449674 100644
--- a/roles/matrix-dimension/tasks/setup_install.yml
+++ b/roles/matrix-dimension/tasks/setup_install.yml
@@ -1,5 +1,32 @@
 ---
 
+- set_fact:
+    matrix_dimension_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_dimension_sqlite_database_path_local }}"
+      register: matrix_dimension_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_dimension_sqlite_database_path_local }}"
+              dst: "{{ matrix_dimension_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_dimension_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-dimension.service']
+              pgloader_options: ['--with "quote identifiers"']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_dimension_requires_restart: true
+      when: "matrix_dimension_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_dimension_database_engine == 'postgres'"
+
 - name: Ensure Dimension base path exists
   file:
     path: "{{ matrix_dimension_base_path }}"
@@ -34,3 +61,9 @@
   service:
     daemon_reload: yes
   when: "matrix_dimension_systemd_service_result.changed|bool"
+
+- name: Ensure matrix-dimension.service restarted, if necessary
+  service:
+    name: "matrix-dimension.service"
+    state: restarted
+  when: "matrix_dimension_requires_restart|bool"
diff --git a/roles/matrix-dimension/templates/config.yaml.j2 b/roles/matrix-dimension/templates/config.yaml.j2
index b7ca1ee3b8d..200871e7e19 100644
--- a/roles/matrix-dimension/templates/config.yaml.j2
+++ b/roles/matrix-dimension/templates/config.yaml.j2
@@ -44,7 +44,11 @@ widgetBlacklist:
 
 # Where the database for Dimension is
 database:
-  uri: "postgres://matrix_dimension:{{ matrix_additional_databases | selectattr('name', 'equalto', 'matrix_dimension') | map(attribute='pass') | first }}@{{ matrix_postgres_connection_hostname }}/matrix_dimension"
+{% if matrix_dimension_database_engine == 'sqlite' %}
+  file: {{ matrix_dimension_sqlite_database_path_in_container|to_json }}
+{% elif matrix_dimension_database_engine == 'postgres' %}
+  uri: {{ matrix_dimension_database_connection_string|to_json }}
+{% endif %}
 
 # Display settings that apply to self-hosted go-neb instances
 goneb:
diff --git a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
index 1c900a162f4..ff10224af71 100644
--- a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
+++ b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2
@@ -16,7 +16,9 @@ ExecStartPre=-{{ matrix_host_command_docker }} kill matrix-dimension
 ExecStartPre=-{{ matrix_host_command_docker }} rm matrix-dimension
 
 # Fixup database ownership if it got changed somehow (during a server migration, etc.)
-ExecStartPre=-{{ matrix_host_command_chown }} {{ matrix_dimension_user_uid }}:{{ matrix_dimension_user_gid }} {{ matrix_dimension_base_path }}/dimension.db
+{% if matrix_dimension_database_engine == 'sqlite' %}
+ExecStartPre=-{{ matrix_host_command_chown }} {{ matrix_dimension_user_uid }}:{{ matrix_dimension_user_gid }} {{ matrix_dimension_sqlite_database_path_local }}
+{% endif %}
 
 ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-dimension \
 			--log-driver=none \

From dd797ba6a76e18f8b2bfa676b6f3e78f18f7cf47 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 22:28:20 +0200
Subject: [PATCH 60/79] Fix Postgres database importing/upgrading conflicts

We were running into conflicts, because having initialized
the roles (users) and databases, trying to import leads to
errors (role XXX already exists, etc.).

We were previously ignoring the Synapse database (`homeserver`)
when upgrading/importing, because that one gets created by default
whenever the container starts.

For our additional databases, it's a similar situation now.
It's not created by default as soon as Postgres starts with an empty
database, but rather we create it as part of running the playbook.

So we either need to skip those role/database creation statements
while upgrading/importing, or to avoid creating the additional database
and rely on the import for that. I've gone for the former, because
it's already similar to what we were doing and it's simpler
(it lets `setup_postgres.yml` be the same in all scenarios).
---
 group_vars/matrix_servers                        | 14 ++++++++++++++
 roles/matrix-postgres/defaults/main.yml          | 16 ++++++++++++++++
 roles/matrix-postgres/tasks/import_postgres.yml  |  4 ++--
 roles/matrix-postgres/tasks/upgrade_postgres.yml |  4 ++--
 4 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index d7163ab7626..932b288ad10 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1100,6 +1100,20 @@ matrix_postgres_additional_databases: |
     }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else [])
    }}
 
+matrix_postgres_import_roles_to_ignore: |
+  {{
+    [matrix_postgres_connection_username]
+    +
+    matrix_postgres_additional_databases|map(attribute='username')
+  }}
+
+matrix_postgres_import_databases_to_ignore: |
+  {{
+    [matrix_postgres_db_name]
+    +
+    matrix_postgres_additional_databases|map(attribute='name')
+  }}
+
 ######################################################################
 #
 # /matrix-postgres
diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 5755742f58a..0c5162812ac 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -43,6 +43,22 @@ matrix_postgres_container_postgres_bind_port: ""
 #     password: some_password
 matrix_postgres_additional_databases: []
 
+# A list of roles/users to avoid creating when importing (or upgrading) the database.
+# If a dump file contains the roles and they've also been created beforehand (see `matrix_postgres_additional_databases`),
+# importing would fail.
+# We either need to not create them or to ignore the `CREATE ROLE` statements in the dump.
+matrix_postgres_import_roles_to_ignore: [matrix_postgres_connection_username]
+
+matrix_postgres_import_roles_ignore_regex: "^CREATE ROLE ({{ matrix_postgres_import_roles_to_ignore|join('|') }});"
+
+# A list of databases to avoid creating when importing (or upgrading) the database.
+# If a dump file contains the databases and they've also been created beforehand (see `matrix_postgres_additional_databases`),
+# importing would fail.
+# We either need to not create them or to ignore the `CREATE DATABASE` statements in the dump.
+matrix_postgres_import_databases_to_ignore: [matrix_postgres_db_name]
+
+matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_postgres_import_databases_to_ignore|join('|') }})\\s"
+
 # The number of seconds to wait after starting `matrix-postgres.service`
 # and before trying to run queries for creating additional databases/users against it.
 #
diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml
index 33d98691dbe..c26affbbb9f 100644
--- a/roles/matrix-postgres/tasks/import_postgres.yml
+++ b/roles/matrix-postgres/tasks/import_postgres.yml
@@ -74,8 +74,8 @@
       {{ matrix_postgres_docker_image_latest }}
       -c "cat /{{ server_path_postgres_dump|basename }} |
       {{ 'gunzip |' if server_path_postgres_dump.endswith('.gz') else '' }}
-      grep -vE '^CREATE ROLE {{ matrix_postgres_connection_username }}' |
-      grep -vE '^CREATE DATABASE {{ matrix_postgres_db_name }}' |
+      grep -vE '{{ matrix_postgres_import_roles_ignore_regex }}' |
+      grep -vE '{{ matrix_postgres_import_databases_ignore_regex }}' |
       psql -v ON_ERROR_STOP=1 -h matrix-postgres"
 
 # This is a hack.
diff --git a/roles/matrix-postgres/tasks/upgrade_postgres.yml b/roles/matrix-postgres/tasks/upgrade_postgres.yml
index 72f327b3551..564265d85b3 100644
--- a/roles/matrix-postgres/tasks/upgrade_postgres.yml
+++ b/roles/matrix-postgres/tasks/upgrade_postgres.yml
@@ -135,8 +135,8 @@
       {{ matrix_postgres_docker_image_latest }}
       -c "cat /in/{{ postgres_dump_name }} |
       {{ 'gunzip |' if postgres_dump_name.endswith('.gz') else '' }}
-      grep -vE '^CREATE ROLE {{ matrix_postgres_connection_username }}' |
-      grep -vE '^CREATE DATABASE {{ matrix_postgres_db_name }}' |
+      grep -vE '{{ matrix_postgres_import_roles_ignore_regex }}' |
+      grep -vE '{{ matrix_postgres_import_databases_ignore_regex }}' |
       psql -v ON_ERROR_STOP=1 -h matrix-postgres"
 
 # This is a hack.

From e2ba46bf013785079eeca380a9926598b9fd1fe4 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Mon, 14 Dec 2020 22:40:37 +0200
Subject: [PATCH 61/79] Fix Jinja2 syntax error (else if -> elif)

---
 roles/matrix-bridge-appservice-irc/defaults/main.yml      | 2 +-
 roles/matrix-bridge-appservice-slack/defaults/main.yml    | 2 +-
 roles/matrix-bridge-mx-puppet-discord/defaults/main.yml   | 2 +-
 roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml | 2 +-
 roles/matrix-bridge-mx-puppet-skype/defaults/main.yml     | 2 +-
 roles/matrix-bridge-mx-puppet-slack/defaults/main.yml     | 2 +-
 roles/matrix-bridge-mx-puppet-steam/defaults/main.yml     | 2 +-
 roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml   | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml
index bf23b0abb05..5110fd852c0 100644
--- a/roles/matrix-bridge-appservice-irc/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml
@@ -26,7 +26,7 @@ matrix_appservice_irc_database_name: matrix_appservice_irc
 matrix_appservice_irc_database_connString: >-2
   {%- if matrix_appservice_irc_database_engine == 'postgres' -%}
   postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable
-  {%- else if matrix_appservice_irc_database_engine == 'nedb' -%}
+  {%- elif matrix_appservice_irc_database_engine == 'nedb' -%}
   {{ matrix_appservice_irc_database_engine }}://{{ matrix_appservice_irc_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml
index e2127c29a81..365493ff21a 100644
--- a/roles/matrix-bridge-appservice-slack/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml
@@ -55,7 +55,7 @@ matrix_appservice_slack_database_file: /data
 matrix_appservice_slack_database_connString: >-2
   {%- if matrix_appservice_slack_database_engine == 'postgres' -%}
   postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable
-  {%- else if matrix_appservice_slack_database_engine == 'nedb' -%}
+  {%- elif matrix_appservice_slack_database_engine == 'nedb' -%}
   {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
index 5a65f33e944..a0ad1c0abdf 100644
--- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
@@ -66,7 +66,7 @@ matrix_mx_puppet_discord_database_file: /data/database.db
 matrix_mx_puppet_discord_database_connString: >-2
   {%- if matrix_mx_puppet_discord_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_discord_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_discord_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_discord_database_engine }}://{{ matrix_mx_puppet_discord_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
index c30f9b8b386..4c40bda4045 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
@@ -56,7 +56,7 @@ matrix_mx_puppet_instagram_database_file: /data/database.db
 matrix_mx_puppet_instagram_database_connString: >-2
   {%- if matrix_mx_puppet_instagram_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_instagram_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_instagram_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_instagram_database_engine }}://{{ matrix_mx_puppet_instagram_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
index 3b7448c66b5..db0f4edea3c 100644
--- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
@@ -64,7 +64,7 @@ matrix_mx_puppet_skype_database_file: /data/database.db
 matrix_mx_puppet_skype_database_connString: >-2
   {%- if matrix_mx_puppet_skype_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_skype_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_skype_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_skype_database_engine }}://{{ matrix_mx_puppet_skype_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
index 5f91beb8db6..2ed72611397 100644
--- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
@@ -68,7 +68,7 @@ matrix_mx_puppet_slack_database_file: /data/database.db
 matrix_mx_puppet_slack_database_connString: >-2
   {%- if matrix_mx_puppet_slack_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_slack_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_slack_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_slack_database_engine }}://{{ matrix_mx_puppet_slack_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
index be343db45ab..77f7804bc9c 100644
--- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
@@ -65,7 +65,7 @@ matrix_mx_puppet_steam_database_file: /data/database.db
 matrix_mx_puppet_steam_database_connString: >-2
   {%- if matrix_mx_puppet_steam_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_steam_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_steam_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_steam_database_engine }}://{{ matrix_mx_puppet_steam_database_file }}
   {%- endif -%}
 
diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
index f536029c0fe..91aa264abd2 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
@@ -71,7 +71,7 @@ matrix_mx_puppet_twitter_database_file: /data/database.db
 matrix_mx_puppet_twitter_database_connString: >-2
   {%- if matrix_mx_puppet_twitter_database_engine == 'postgres' -%}
   postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable
-  {%- else if matrix_mx_puppet_twitter_database_engine == 'sqlite' -%}
+  {%- elif matrix_mx_puppet_twitter_database_engine == 'sqlite' -%}
   {{ matrix_mx_puppet_twitter_database_engine }}://{{ matrix_mx_puppet_twitter_database_file }}
   {%- endif -%}
 

From 1bd5c240e582bfc4d4d1fcaaea5be326f5186e92 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 15 Dec 2020 23:18:29 +0200
Subject: [PATCH 62/79] Add support for executing additional DB migration
 statements

In cases where pgloader is not enough and we need to do some additional
migration work after it, we can now use
`additional_psql_statements_list` and
`additional_psql_statements_db_name`.

This is to be used when migrating `matrix-registration`'s data at the
very least.
---
 .../tasks/util/migrate_db_to_postgres.yml     | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index 5d917538e3d..d01611ef290 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -99,6 +99,29 @@
   command:
     cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup"
 
+- block:
+    # We can't use `{{ role_path }}` here, neither with `import_tasks`, nor with `include_tasks`,
+    # because it refers to the role that included this util, and not to the role this file belongs to.
+    - import_tasks: "roles/matrix-postgres/tasks/util/detect_existing_postgres_version.yml"
+
+    - set_fact:
+        matrix_postgres_docker_image_to_use: "{{ matrix_postgres_docker_image_latest if matrix_postgres_detected_version_corresponding_docker_image == '' else matrix_postgres_detected_version_corresponding_docker_image }}"
+
+    - name: Execute additional Postgres SQL migration statements
+      command:
+        cmd: >-
+          {{ matrix_host_command_docker }} run
+          --rm
+          --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+          --cap-drop=ALL
+          --env-file={{ matrix_postgres_base_path }}/env-postgres-psql
+          --network={{ matrix_docker_network }}
+          {{ matrix_postgres_docker_image_to_use }}
+          psql --host=matrix-postgres --dbname={{ matrix_postgres_db_migration_request.additional_psql_statements_db_name }} --command='{{ item }}'
+      with_items: "{{ matrix_postgres_db_migration_request.additional_psql_statements_list }}"
+
+  when: "matrix_postgres_db_migration_request.additional_psql_statements_list|default([])|length > 0"
+
 - name: Inject result
   set_fact:
     matrix_playbook_runtime_results: |

From a197968b7f95c6f7e6d5f3882cb25e1dc1892124 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 15 Dec 2020 23:19:56 +0200
Subject: [PATCH 63/79] Make matrix-registration use Postgres by default

Now that 0.7.2 is out, the Docker image supports Postgres
and we can do the (SQLite -> Postgres) migration.

I've also found out that we needed to fix up the `tokens.ex_date` column
data type a bit to prevent matrix-registration from raising exceptions
when comparing `datetime.now()` with `ex_date` coming from the database.

Example:

> File "/usr/local/lib/python3.8/site-packages/matrix_registration/tokens.py", line 58, in valid
> expired = self.ex_date < datetime.now()
> TypeError: can't compare offset-naive and offset-aware datetimes
---
 group_vars/matrix_servers                           | 5 ++---
 roles/matrix-registration/defaults/main.yml         | 1 -
 roles/matrix-registration/tasks/setup_install.yml   | 5 +++++
 roles/matrix-registration/tasks/validate_config.yml | 7 -------
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 932b288ad10..fda40efdf84 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1317,9 +1317,8 @@ matrix_registration_systemd_required_services_list: |
     (['matrix-postgres.service'] if matrix_postgres_enabled else [])
   }}
 
-# We'd like to use 'postgres' if matrix_postgres_enabled, but the container image doesn't seem to support that.
-# Learn more here: https://github.com/ZerataX/matrix-registration/issues/44
-matrix_registration_database_engine: 'sqlite'
+# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
+matrix_registration_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
 matrix_registration_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx.registr.db') | to_uuid }}"
 
 ######################################################################
diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml
index b39f02c461e..d85faf89a9d 100644
--- a/roles/matrix-registration/defaults/main.yml
+++ b/roles/matrix-registration/defaults/main.yml
@@ -88,7 +88,6 @@ matrix_registration_admin_secret: ""
 
 matrix_registration_riot_instance: "https://riot.im/app/"
 
-
 # Default matrix-registration configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
 #
diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml
index 489bab8ba80..708cb1df561 100644
--- a/roles/matrix-registration/tasks/setup_install.yml
+++ b/roles/matrix-registration/tasks/setup_install.yml
@@ -18,6 +18,11 @@
               engine_variable_name: 'matrix_registration_database_engine'
               engine_old: 'sqlite'
               systemd_services_to_stop: ['matrix-registration.service']
+              # pgloader makes `ex_date` of type `TIMESTAMP WITH TIMEZONE`,
+              # which makes matrix-registration choke on it later on when comparing dates.
+              additional_psql_statements_list:
+                - ALTER TABLE tokens ALTER COLUMN ex_date TYPE TIMESTAMP WITHOUT TIME ZONE;
+              additional_psql_statements_db_name: "{{ matrix_registration_database_db_name }}"
 
         - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 
diff --git a/roles/matrix-registration/tasks/validate_config.yml b/roles/matrix-registration/tasks/validate_config.yml
index 80293bcba3a..90466b46c48 100644
--- a/roles/matrix-registration/tasks/validate_config.yml
+++ b/roles/matrix-registration/tasks/validate_config.yml
@@ -18,10 +18,3 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_registration_docker_repo', 'new': 'matrix_registration_container_image_self_build_repo'}
-
-- name: Fail if Postgres usage attempted
-  fail:
-    msg: >
-      matrix-registration doesn't support using Postgres just yet.
-      Learn more here: https://github.com/ZerataX/matrix-registration/issues/44
-  when: "matrix_registration_database_engine == 'postgres'"

From 6488e11d692d0a44fc5c53e81d986ada35c92be4 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 10:52:36 +0200
Subject: [PATCH 64/79] Relocate some tasks

---
 .../tasks/setup_install.yml                   | 34 +++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
index c3f2b01fbd7..9022f309a81 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
@@ -8,6 +8,23 @@
       The matrix-bridge-mx-puppet-discord role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- name: Check if an old database file already exists
+  stat:
+    path: "{{ matrix_mx_puppet_discord_base_path }}/database.db"
+  register: matrix_mx_puppet_discord_stat_database
+
+- block:
+    - name: (Data relocation) Ensure matrix-mx-puppet-discord.service is stopped
+      service:
+        name: matrix-mx-puppet-discord
+        state: stopped
+        daemon_reload: yes
+      failed_when: False
+
+    - name: (Data relocation) Move mx-puppet-discord database file to ./data directory
+      command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db"
+  when: "matrix_mx_puppet_discord_stat_database.stat.exists"
+
 - name: Ensure MX Puppet Discord image is pulled
   docker_image:
     name: "{{ matrix_mx_puppet_discord_docker_image }}"
@@ -49,23 +66,6 @@
       pull: yes
   when: "matrix_mx_puppet_discord_enabled|bool and matrix_mx_puppet_discord_container_image_self_build|bool"
 
-- name: Check if an old database file already exists
-  stat:
-    path: "{{ matrix_mx_puppet_discord_base_path }}/database.db"
-  register: matrix_mx_puppet_discord_stat_database
-
-- name: (Data relocation) Ensure matrix-mx-puppet-discord.service is stopped
-  service:
-    name: matrix-mx-puppet-discord
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
-  when: "matrix_mx_puppet_discord_stat_database.stat.exists"
-
-- name: (Data relocation) Move mx-puppet-discord database file to ./data directory
-  command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db"
-  when: "matrix_mx_puppet_discord_stat_database.stat.exists"
-
 - name: Ensure mx-puppet-discord config.yaml installed
   copy:
     content: "{{ matrix_mx_puppet_discord_configuration|to_nice_yaml }}"

From 9b4bf7358784f4f7df163cbf922e643dca5f0977 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 11:08:07 +0200
Subject: [PATCH 65/79] Fix undefined variable reference

---
 roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
index c24c7e54cda..27f0c5673cf 100644
--- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
@@ -100,7 +100,7 @@ namePatterns:
   group: :name
 
 database:
-{% if matrix_mx_puppet_slack_discord_database_engine == 'sqlite' %}
+{% if matrix_mx_puppet_discord_database_engine == 'sqlite' %}
   # Use SQLite3 as a database backend
   # The name of the database file
   filename: /data/database.db

From 149872e00c59960494877bac032b9a4d0a5d40ea Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 11:10:10 +0200
Subject: [PATCH 66/79] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-discord

---
 .../defaults/main.yml                         | 15 +++++------
 .../tasks/setup_install.yml                   | 26 +++++++++++++++++++
 .../tasks/validate_config.yml                 | 20 --------------
 .../templates/config.yaml.j2                  |  4 +--
 4 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
index a0ad1c0abdf..ad86a2416a9 100644
--- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
@@ -57,18 +57,17 @@ matrix_mx_puppet_discord_login_shared_secret: ''
 
 # Database configuration
 matrix_mx_puppet_discord_database_engine: 'sqlite'
+
+matrix_mx_puppet_discord_sqlite_database_path_local: "{{ matrix_mx_puppet_discord_data_path }}/database.db"
+matrix_mx_puppet_discord_sqlite_database_path_in_container: "/data/database.db"
+
 matrix_mx_puppet_discord_database_username: matrix_mx_puppet_discord
 matrix_mx_puppet_discord_database_password: ~
 matrix_mx_puppet_discord_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_discord_database_port: 5432
-matrix_mx_puppet_discord_database_name: matrix_mx_puppet_discord
-matrix_mx_puppet_discord_database_file: /data/database.db
-matrix_mx_puppet_discord_database_connString: >-2
-  {%- if matrix_mx_puppet_discord_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_discord_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_discord_database_engine }}://{{ matrix_mx_puppet_discord_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_discord_database_db_name: matrix_mx_puppet_discord
+
+matrix_mx_puppet_discord_database_connection_string: 'postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
index 9022f309a81..08a199c4ed4 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
@@ -25,6 +25,32 @@
       command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db"
   when: "matrix_mx_puppet_discord_stat_database.stat.exists"
 
+- set_fact:
+    matrix_mx_puppet_discord_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_discord_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_discord_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_discord_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_discord_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_discord_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-discord.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_discord_requires_restart: true
+      when: "matrix_mx_puppet_discord_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_discord_database_engine == 'postgres'"
+
 - name: Ensure MX Puppet Discord image is pulled
   docker_image:
     name: "{{ matrix_mx_puppet_discord_docker_image }}"
diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
index 8db47ed1f92..c253eda296e 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/validate_config.yml
@@ -8,23 +8,3 @@
   with_items:
     - "matrix_mx_puppet_discord_appservice_token"
     - "matrix_mx_puppet_discord_homeserver_token"
-
-- block:
-  - name: Check if a SQLite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_discord_data_path }}/database.db"
-    register: matrix_mx_puppet_discord_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-2
-        matrix_mx_puppet_discord_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_discord_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_discord_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_discord_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_discord_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_discord_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_discord_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
index 27f0c5673cf..1f4548d8353 100644
--- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2
@@ -103,14 +103,14 @@ database:
 {% if matrix_mx_puppet_discord_database_engine == 'sqlite' %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: /data/database.db
+  filename: {{ matrix_mx_puppet_discord_sqlite_database_path_in_container|to_json }}
 {% else %}
   # Use Postgres as a database backend
   # If set, will be used instead of SQLite3
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_discord_database_connString|to_json }}
+  connString: {{ matrix_mx_puppet_discord_database_connection_string|to_json }}
 {% endif %}
 
 logging:

From e64758c11951b7d2720431b139be0edc454085bc Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 13:24:53 +0200
Subject: [PATCH 67/79] Add missing restart task

Should have been part of 149872e00c5
---
 .../matrix-bridge-mx-puppet-discord/tasks/setup_install.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
index 08a199c4ed4..9801bf828be 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
@@ -119,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mx_puppet_discord_systemd_service_result.changed"
+
+- name: Ensure matrix-mx-puppet-discord.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-discord.service"
+    state: restarted
+  when: "matrix_mx_puppet_discord_requires_restart|bool"

From 44c9f4daca05fca27aa1fe8399349a34a9837015 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 13:30:52 +0200
Subject: [PATCH 68/79] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-instagram

---
 .../defaults/main.yml                         | 15 ++++-----
 .../tasks/setup_install.yml                   | 33 ++++++++++++++++++-
 .../tasks/validate_config.yml                 | 20 -----------
 .../templates/config.yaml.j2                  |  4 +--
 4 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
index 4c40bda4045..61bf3dcfeac 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
@@ -47,18 +47,17 @@ matrix_mx_puppet_instagram_homeserver_token: ''
 matrix_mx_puppet_instagram_login_shared_secret: ''
 
 matrix_mx_puppet_instagram_database_engine: sqlite
+
+matrix_mx_puppet_instagram_sqlite_database_path_local: "{{ matrix_mx_puppet_instagram_data_path }}/database.db"
+matrix_mx_puppet_instagram_sqlite_database_path_in_container: "/data/database.db"
+
 matrix_mx_puppet_instagram_database_username: matrix_mx_puppet_instagram
 matrix_mx_puppet_instagram_database_password: ~
 matrix_mx_puppet_instagram_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_instagram_database_port: 5432
-matrix_mx_puppet_instagram_database_name: matrix_mx_puppet_instagram
-matrix_mx_puppet_instagram_database_file: /data/database.db
-matrix_mx_puppet_instagram_database_connString: >-2
-  {%- if matrix_mx_puppet_instagram_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_instagram_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_instagram_database_engine }}://{{ matrix_mx_puppet_instagram_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_instagram_database_db_name: matrix_mx_puppet_instagram
+
+matrix_mx_puppet_instagram_database_connection_string: 'postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml
index 78bd78c1697..76bbd629b80 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml
@@ -8,6 +8,33 @@
       The matrix-bridge-mx-puppet-instagram role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+
+- set_fact:
+    matrix_mx_puppet_instagram_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_instagram_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_instagram_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_instagram_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_instagram_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-instagram.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_instagram_requires_restart: true
+      when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_instagram_database_engine == 'postgres'"
+
 - name: Ensure mx-puppet-instagram image is pulled
   docker_image:
     name: "{{ matrix_mx_puppet_instagram_docker_image }}"
@@ -77,4 +104,8 @@
     daemon_reload: yes
   when: "matrix_mx_puppet_instagram_systemd_service_result.changed"
 
-
+- name: Ensure matrix-mx-puppet-instagram.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-instagram.service"
+    state: restarted
+  when: "matrix_mx_puppet_instagram_requires_restart|bool"
diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
index c2115cbb2e2..b6d9d994e3a 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/validate_config.yml
@@ -8,23 +8,3 @@
   with_items:
     - "matrix_mx_puppet_instagram_appservice_token"
     - "matrix_mx_puppet_instagram_homeserver_token"
-
-- block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_instagram_data_path }}/database.db"
-    register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mx_puppet_instagram_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_instagram_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_instagram_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_instagram_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_instagram_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_instagram_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
index d89c5bdb23d..b830da2b682 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2
@@ -50,11 +50,11 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_instagram_database_connString | to_json }}
+  connString: {{ matrix_mx_puppet_instagram_database_connection_string|to_json }}
 {% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: {{ matrix_mx_puppet_instagram_database_file }}
+  filename: {{ matrix_mx_puppet_instagram_sqlite_database_path_in_container|to_json }}
 {% endif %}
 
 logging:

From d135cd9cd3117ca659459db626d8ed942d474325 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 13:44:33 +0200
Subject: [PATCH 69/79] Ensure mx-puppet-discord directories are created before
 attempting migration

Our old (base-path -> data-path) SQLite migration can't work otherwise.

It's probably not necessary to keep it anymore, but since we still do,
at least we should take care to ensure it works.
---
 .../tasks/setup_install.yml                   | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
index 9801bf828be..3e3b2f9447e 100644
--- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml
@@ -8,6 +8,20 @@
       The matrix-bridge-mx-puppet-discord role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
+- name: Ensure MX Puppet Discord paths exist
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    mode: 0750
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_groupname }}"
+  with_items:
+    - { path: "{{ matrix_mx_puppet_discord_base_path }}", when: true }
+    - { path: "{{ matrix_mx_puppet_discord_config_path }}", when: true }
+    - { path: "{{ matrix_mx_puppet_discord_data_path }}", when: true }
+    - { path: "{{ matrix_mx_puppet_discord_docker_src_files_path }}", when: "{{ matrix_mx_puppet_discord_container_image_self_build }}" }
+  when: matrix_mx_puppet_discord_enabled|bool and item.when|bool
+
 - name: Check if an old database file already exists
   stat:
     path: "{{ matrix_mx_puppet_discord_base_path }}/database.db"
@@ -59,20 +73,6 @@
     force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_discord_docker_image_force_pull }}"
   when: matrix_mx_puppet_discord_enabled|bool and not matrix_mx_puppet_discord_container_image_self_build
 
-- name: Ensure MX Puppet Discord paths exist
-  file:
-    path: "{{ item.path }}"
-    state: directory
-    mode: 0750
-    owner: "{{ matrix_user_username }}"
-    group: "{{ matrix_user_groupname }}"
-  with_items:
-    - { path: "{{ matrix_mx_puppet_discord_base_path }}", when: true }
-    - { path: "{{ matrix_mx_puppet_discord_config_path }}", when: true }
-    - { path: "{{ matrix_mx_puppet_discord_data_path }}", when: true }
-    - { path: "{{ matrix_mx_puppet_discord_docker_src_files_path }}", when: "{{ matrix_mx_puppet_discord_container_image_self_build }}" }
-  when: matrix_mx_puppet_discord_enabled|bool and item.when|bool
-
 - name: Ensure MX Puppet Discord repository is present on self build
   git:
     repo: "{{ matrix_mx_puppet_discord_container_image_self_build_repo }}"

From e49eb078a25bab1e320ac6d5c56f58461f8b19ad Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 16:29:47 +0200
Subject: [PATCH 70/79] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-skype

---
 .../defaults/main.yml                         | 15 ++--
 .../tasks/setup_install.yml                   | 82 +++++++++++++------
 .../tasks/validate_config.yml                 | 20 -----
 .../templates/config.yaml.j2                  |  4 +-
 4 files changed, 66 insertions(+), 55 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
index db0f4edea3c..f3e17a593a2 100644
--- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
@@ -55,18 +55,17 @@ matrix_mx_puppet_skype_login_shared_secret: ''
 
 # Database configuration, role default is `sqlite` but playbook default is `postgres`
 matrix_mx_puppet_skype_database_engine: sqlite
+
+matrix_mx_puppet_skype_sqlite_database_path_local: "{{ matrix_mx_puppet_skype_data_path }}/database.db"
+matrix_mx_puppet_skype_sqlite_database_path_in_container: "/data/database.db"
+
 matrix_mx_puppet_skype_database_username: matrix_mx_puppet_skype
 matrix_mx_puppet_skype_database_password: ~
 matrix_mx_puppet_skype_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_skype_database_port: 5432
-matrix_mx_puppet_skype_database_name: matrix_mx_puppet_skype
-matrix_mx_puppet_skype_database_file: /data/database.db
-matrix_mx_puppet_skype_database_connString: >-2
-  {%- if matrix_mx_puppet_skype_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_skype_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_skype_database_engine }}://{{ matrix_mx_puppet_skype_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_skype_database_db_name: matrix_mx_puppet_skype
+
+matrix_mx_puppet_skype_database_connection_string: 'postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml
index fb5185eda22..9289a793413 100644
--- a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml
@@ -8,14 +8,6 @@
       The matrix-bridge-mx-puppet-skype role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
-- name: Ensure MX Puppet Skype image is pulled
-  docker_image:
-    name: "{{ matrix_mx_puppet_skype_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_mx_puppet_skype_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_skype_docker_image_force_pull }}"
-  when: matrix_mx_puppet_skype_enabled|bool and not matrix_mx_puppet_skype_container_image_self_build
-
 - name: Ensure MX Puppet Skype paths exist
   file:
     path: "{{ item.path }}"
@@ -30,6 +22,57 @@
     - { path: "{{ matrix_mx_puppet_skype_docker_src_files_path }}", when: "{{ matrix_mx_puppet_skype_container_image_self_build }}" }
   when: matrix_mx_puppet_skype_enabled|bool and item.when|bool
 
+- name: Check if an old database file already exists
+  stat:
+    path: "{{ matrix_mx_puppet_skype_base_path }}/database.db"
+  register: matrix_mx_puppet_skype_stat_database
+
+- name: (Data relocation) Ensure matrix-mx-puppet-skype.service is stopped
+  service:
+    name: matrix-mx-puppet-skype
+    state: stopped
+    daemon_reload: yes
+  failed_when: false
+  when: "matrix_mx_puppet_skype_stat_database.stat.exists"
+
+- name: (Data relocation) Move mx-puppet-skype database file to ./data directory
+  command: "mv {{ matrix_mx_puppet_skype_base_path }}/database.db {{ matrix_mx_puppet_skype_data_path }}/database.db"
+  when: "matrix_mx_puppet_skype_stat_database.stat.exists"
+
+- set_fact:
+    matrix_mx_puppet_skype_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_skype_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_skype_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_skype_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_skype_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_skype_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-skype.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_skype_requires_restart: true
+      when: "matrix_mx_puppet_skype_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_skype_database_engine == 'postgres'"
+
+- name: Ensure MX Puppet Skype image is pulled
+  docker_image:
+    name: "{{ matrix_mx_puppet_skype_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mx_puppet_skype_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_skype_docker_image_force_pull }}"
+  when: matrix_mx_puppet_skype_enabled|bool and not matrix_mx_puppet_skype_container_image_self_build
+
 - name: Ensure MX Puppet Skype repository is present on self build
   git:
     repo: "{{ matrix_mx_puppet_skype_container_image_self_build_repo }}"
@@ -49,23 +92,6 @@
       pull: yes
   when: "matrix_mx_puppet_skype_enabled|bool and matrix_mx_puppet_skype_container_image_self_build|bool"
 
-- name: Check if an old database file already exists
-  stat:
-    path: "{{ matrix_mx_puppet_skype_base_path }}/database.db"
-  register: matrix_mx_puppet_skype_stat_database
-
-- name: (Data relocation) Ensure matrix-mx-puppet-skype.service is stopped
-  service:
-    name: matrix-mx-puppet-skype
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
-  when: "matrix_mx_puppet_skype_stat_database.stat.exists"
-
-- name: (Data relocation) Move mx-puppet-skype database file to ./data directory
-  command: "mv {{ matrix_mx_puppet_skype_base_path }}/database.db {{ matrix_mx_puppet_skype_data_path }}/database.db"
-  when: "matrix_mx_puppet_skype_stat_database.stat.exists"
-
 - name: Ensure mx-puppet-skype config.yaml installed
   copy:
     content: "{{ matrix_mx_puppet_skype_configuration|to_nice_yaml }}"
@@ -93,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mx_puppet_skype_systemd_service_result.changed"
+
+- name: Ensure matrix-mx-puppet-skype.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-skype.service"
+    state: restarted
+  when: "matrix_mx_puppet_skype_requires_restart|bool"
diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
index c7100e51ab2..7ed433b12f7 100644
--- a/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/tasks/validate_config.yml
@@ -8,23 +8,3 @@
   with_items:
     - "matrix_mx_puppet_skype_appservice_token"
     - "matrix_mx_puppet_skype_homeserver_token"
-
-- block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_skype_data_path }}/database.db"
-    register: matrix_mx_puppet_skype_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mx_puppet_skype_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_skype_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_skype_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_skype_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_skype_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_skype_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_skype_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
index a32523b1cda..d41d3a23568 100644
--- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2
@@ -74,11 +74,11 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_skype_database_connString | to_json }}
+  connString: {{ matrix_mx_puppet_skype_database_connection_string|to_json }}
 {% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: {{ matrix_mx_puppet_skype_database_file }}
+  filename: {{ matrix_mx_puppet_skype_sqlite_database_path_in_container|to_json }}
 {% endif %}
 
 provisioning:

From 262a25f997a032826b0348f3d966bcac55b9959e Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 16:39:21 +0200
Subject: [PATCH 71/79] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-slack

---
 .../defaults/main.yml                         | 15 ++--
 .../tasks/setup_install.yml                   | 74 +++++++++++++------
 .../tasks/validate_config.yml                 | 20 -----
 .../templates/config.yaml.j2                  |  4 +-
 4 files changed, 62 insertions(+), 51 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
index 2ed72611397..9aad602a4ba 100644
--- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
@@ -59,18 +59,17 @@ matrix_mx_puppet_slack_login_shared_secret: ''
 
 # Database configuration, role uses 'sqlite' per default but playbook sets up postgres by default
 matrix_mx_puppet_slack_database_engine: sqlite
+
+matrix_mx_puppet_slack_sqlite_database_path_local: "{{ matrix_mx_puppet_slack_data_path }}/database.db"
+matrix_mx_puppet_slack_sqlite_database_path_in_container: "/data/database.db"
+
 matrix_mx_puppet_slack_database_username: matrix_mx_puppet_slack
 matrix_mx_puppet_slack_database_password: ~
 matrix_mx_puppet_slack_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_slack_database_port: 5432
-matrix_mx_puppet_slack_database_name: matrix_mx_puppet_slack
-matrix_mx_puppet_slack_database_file: /data/database.db
-matrix_mx_puppet_slack_database_connString: >-2
-  {%- if matrix_mx_puppet_slack_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_slack_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_slack_database_engine }}://{{ matrix_mx_puppet_slack_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_slack_database_db_name: matrix_mx_puppet_slack
+
+matrix_mx_puppet_slack_database_connection_string: 'postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml
index 2e45ecf6070..5d68435cdad 100644
--- a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml
@@ -8,14 +8,6 @@
       The matrix-bridge-mx-puppet-slack role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
-- name: Ensure MX Puppet Slack image is pulled
-  docker_image:
-    name: "{{ matrix_mx_puppet_slack_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_mx_puppet_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_slack_docker_image_force_pull }}"
-  when: matrix_mx_puppet_slack_enabled|bool and not matrix_mx_puppet_slack_container_image_self_build
-
 - name: Ensure MX Puppet Slack paths exist
   file:
     path: "{{ item.path }}"
@@ -30,6 +22,53 @@
     - { path: "{{ matrix_mx_puppet_slack_docker_src_files_path }}", when: "{{ matrix_mx_puppet_slack_container_image_self_build }}" }
   when: matrix_mx_puppet_slack_enabled|bool and item.when|bool
 
+- name: Check if an old database file already exists
+  stat:
+    path: "{{ matrix_mx_puppet_slack_base_path }}/database.db"
+  register: matrix_mx_puppet_slack_stat_database
+
+- name: (Data relocation) Ensure matrix-mx-puppet-slack.service is stopped
+  service:
+    name: matrix-mx-puppet-slack
+    state: stopped
+    daemon_reload: yes
+  failed_when: false
+  when: "matrix_mx_puppet_slack_stat_database.stat.exists"
+
+- set_fact:
+    matrix_mx_puppet_slack_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_slack_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_slack_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_slack_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_slack_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_slack_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-slack.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_slack_requires_restart: true
+      when: "matrix_mx_puppet_slack_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_slack_database_engine == 'postgres'"
+
+- name: Ensure MX Puppet Slack image is pulled
+  docker_image:
+    name: "{{ matrix_mx_puppet_slack_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mx_puppet_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_slack_docker_image_force_pull }}"
+  when: matrix_mx_puppet_slack_enabled|bool and not matrix_mx_puppet_slack_container_image_self_build
+
 - name: Ensure MX Puppet Slack repository is present on self build
   git:
     repo: "{{ matrix_mx_puppet_slack_container_image_self_build_repo }}"
@@ -49,19 +88,6 @@
       pull: yes
   when: "matrix_mx_puppet_slack_enabled|bool and matrix_mx_puppet_slack_container_image_self_build"
 
-- name: Check if an old database file already exists
-  stat:
-    path: "{{ matrix_mx_puppet_slack_base_path }}/database.db"
-  register: matrix_mx_puppet_slack_stat_database
-
-- name: (Data relocation) Ensure matrix-mx-puppet-slack.service is stopped
-  service:
-    name: matrix-mx-puppet-slack
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
-  when: "matrix_mx_puppet_slack_stat_database.stat.exists"
-
 - name: (Data relocation) Move mx-puppet-slack database file to ./data directory
   command: "mv {{ matrix_mx_puppet_slack_base_path }}/database.db {{ matrix_mx_puppet_slack_data_path }}/database.db"
   when: "matrix_mx_puppet_slack_stat_database.stat.exists"
@@ -93,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mx_puppet_slack_systemd_service_result.changed"
+
+- name: Ensure matrix-mx-puppet-slack.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-slack.service"
+    state: restarted
+  when: "matrix_mx_puppet_slack_requires_restart|bool"
diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
index bc722ee63b6..3a0bca11144 100644
--- a/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/tasks/validate_config.yml
@@ -8,23 +8,3 @@
   with_items:
     - "matrix_mx_puppet_slack_appservice_token"
     - "matrix_mx_puppet_slack_homeserver_token"
-
-- block:
-  - name: Check if sqlite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_slack_data_path }}/database.db"
-    register: matrix_mx_puppet_slack_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mx_puppet_slack_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_slack_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_slack_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_slack_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_slack_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_slack_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_slack_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
index 58c5c11abe2..af6b5cb88db 100644
--- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2
@@ -64,11 +64,11 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_slack_database_connString | to_json }}
+  connString: {{ matrix_mx_puppet_slack_database_connection_string|to_json }}
 {% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: {{ matrix_mx_puppet_slack_database_file }}
+  filename: {{ matrix_mx_puppet_slack_sqlite_database_path_in_container|to_json }}
 {% endif %}
 
 logging:

From 69cc2145d2c710d3874fe2ec12315266561e10a1 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 16:51:59 +0200
Subject: [PATCH 72/79] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-steam

---
 group_vars/matrix_servers                     |  2 +-
 .../defaults/main.yml                         | 15 ++--
 .../tasks/setup_install.yml                   | 82 +++++++++++++------
 .../tasks/validate_config.yml                 | 22 -----
 .../templates/config.yaml.j2                  |  4 +-
 5 files changed, 67 insertions(+), 58 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index fda40efdf84..283d5b6e3b1 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -583,7 +583,7 @@ matrix_mx_puppet_steam_homeserver_token: "{{ matrix_synapse_macaroon_secret_key
 matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}"
 
 # Postgres is the default, except if not using `matrix_postgres` (internal postgres)
-matrix_mx_puppet_stream_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_mx_puppet_steam_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
 matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}"
 
 ######################################################################
diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
index 77f7804bc9c..c1f6a385584 100644
--- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
@@ -56,18 +56,17 @@ matrix_mx_puppet_steam_homeserver_token: ''
 matrix_mx_puppet_steam_login_shared_secret: ''
 
 matrix_mx_puppet_steam_database_engine: sqlite
+
+matrix_mx_puppet_steam_sqlite_database_path_local: "{{ matrix_mx_puppet_steam_data_path }}/database.db"
+matrix_mx_puppet_steam_sqlite_database_path_in_container: "/data/database.db"
+
 matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam
 matrix_mx_puppet_steam_database_password: ~
 matrix_mx_puppet_steam_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_steam_database_port: 5432
-matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam
-matrix_mx_puppet_steam_database_file: /data/database.db
-matrix_mx_puppet_steam_database_connString: >-2
-  {%- if matrix_mx_puppet_steam_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_steam_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_steam_database_engine }}://{{ matrix_mx_puppet_steam_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_steam_database_db_name: matrix_mx_puppet_steam
+
+matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml
index 7b7f8211581..71f6d8897f9 100644
--- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml
@@ -8,14 +8,6 @@
       The matrix-bridge-mx-puppet-steam role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
-- name: Ensure MX Puppet Steam image is pulled
-  docker_image:
-    name: "{{ matrix_mx_puppet_steam_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_steam_docker_image_force_pull }}"
-  when: matrix_mx_puppet_steam_enabled|bool and not matrix_mx_puppet_steam_container_image_self_build
-
 - name: Ensure MX Puppet Steam paths exist
   file:
     path: "{{ item.path }}"
@@ -30,6 +22,57 @@
     - { path: "{{ matrix_mx_puppet_steam_docker_src_files_path }}", when: "{{ matrix_mx_puppet_steam_container_image_self_build }}" }
   when: matrix_mx_puppet_steam_enabled|bool and item.when|bool
 
+- name: Check if an old database file already exists
+  stat:
+    path: "{{ matrix_mx_puppet_steam_base_path }}/database.db"
+  register: matrix_mx_puppet_steam_stat_database
+
+- name: (Data relocation) Ensure matrix-mx-puppet-steam.service is stopped
+  service:
+    name: matrix-mx-puppet-steam
+    state: stopped
+    daemon_reload: yes
+  failed_when: false
+  when: "matrix_mx_puppet_steam_stat_database.stat.exists"
+
+- name: (Data relocation) Move mx-puppet-steam database file to ./data directory
+  command: "mv {{ matrix_mx_puppet_steam_base_path }}/database.db {{ matrix_mx_puppet_steam_data_path }}/database.db"
+  when: "matrix_mx_puppet_steam_stat_database.stat.exists"
+
+- set_fact:
+    matrix_mx_puppet_steam_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_steam_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_steam_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_steam_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_steam_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-steam.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_steam_requires_restart: true
+      when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_steam_database_engine == 'postgres'"
+
+- name: Ensure MX Puppet Steam image is pulled
+  docker_image:
+    name: "{{ matrix_mx_puppet_steam_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_steam_docker_image_force_pull }}"
+  when: matrix_mx_puppet_steam_enabled|bool and not matrix_mx_puppet_steam_container_image_self_build
+
 - name: Ensure MX Puppet Steam repository is present on self build
   git:
     repo: "{{ matrix_mx_puppet_steam_container_image_self_build_repo }}"
@@ -49,23 +92,6 @@
       pull: yes
   when: "matrix_mx_puppet_steam_enabled|bool and matrix_mx_puppet_steam_container_image_self_build"
 
-- name: Check if an old database file already exists
-  stat:
-    path: "{{ matrix_mx_puppet_steam_base_path }}/database.db"
-  register: matrix_mx_puppet_steam_stat_database
-
-- name: (Data relocation) Ensure matrix-mx-puppet-steam.service is stopped
-  service:
-    name: matrix-mx-puppet-steam
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
-  when: "matrix_mx_puppet_steam_stat_database.stat.exists"
-
-- name: (Data relocation) Move mx-puppet-steam database file to ./data directory
-  command: "mv {{ matrix_mx_puppet_steam_base_path }}/database.db {{ matrix_mx_puppet_steam_data_path }}/database.db"
-  when: "matrix_mx_puppet_steam_stat_database.stat.exists"
-
 - name: Ensure mx-puppet-steam config.yaml installed
   copy:
     content: "{{ matrix_mx_puppet_steam_configuration|to_nice_yaml }}"
@@ -93,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mx_puppet_steam_systemd_service_result.changed"
+
+- name: Ensure matrix-mx-puppet-steam.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-steam.service"
+    state: restarted
+  when: "matrix_mx_puppet_steam_requires_restart|bool"
diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
index 50ce15d9de0..a8bc6a420b0 100644
--- a/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/tasks/validate_config.yml
@@ -8,25 +8,3 @@
   with_items:
     - "matrix_mx_puppet_steam_appservice_token"
     - "matrix_mx_puppet_steam_homeserver_token"
-
-- block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_steam_data_path }}"
-    register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mx_puppet_steam_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_steam_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_steam_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_steam_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_steam_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_steam_database_engine == 'postgres'"
-
-
diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
index 8eda278dd52..149e08b69a3 100644
--- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2
@@ -67,11 +67,11 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_steam_database_connString | to_json }}
+  connString: {{ matrix_mx_puppet_steam_database_connection_string|to_json }}
 {% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: {{ matrix_mx_puppet_steam_database_file }}
+  filename: {{ matrix_mx_puppet_steam_sqlite_database_path_in_container|to_json }}
 {% endif %}
 
 logging:

From ab6563ce4e9273cc4e45fbf1b50339c10aff9452 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 17:09:08 +0200
Subject: [PATCH 73/79] Add support for automatic (Postgres -> SQLite)
 migration to mx-puppet-twitter

---
 .../defaults/main.yml                         | 17 ++--
 .../tasks/setup_install.yml                   | 82 +++++++++++++------
 .../tasks/validate_config.yml                 | 22 -----
 .../templates/config.yaml.j2                  |  4 +-
 4 files changed, 67 insertions(+), 58 deletions(-)

diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
index 91aa264abd2..da7a9ae7f48 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
@@ -62,18 +62,17 @@ matrix_mx_puppet_twitter_login_shared_secret: ''
 
 # Database configuration
 matrix_mx_puppet_twitter_database_engine: sqlite
-matrix_mx_puppet_twitter_database_username: matrix:mx_puppet_twitter
+
+matrix_mx_puppet_twitter_sqlite_database_path_local: "{{ matrix_mx_puppet_twitter_data_path }}/database.db"
+matrix_mx_puppet_twitter_sqlite_database_path_in_container: "/data/database.db"
+
+matrix_mx_puppet_twitter_database_username: mx_puppet_twitter
 matrix_mx_puppet_twitter_database_password: ~
 matrix_mx_puppet_twitter_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_twitter_database_port: 5432
-matrix_mx_puppet_twitter_database_name: matrix_mx_puppet_twitter
-matrix_mx_puppet_twitter_database_file: /data/database.db
-matrix_mx_puppet_twitter_database_connString: >-2
-  {%- if matrix_mx_puppet_twitter_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable
-  {%- elif matrix_mx_puppet_twitter_database_engine == 'sqlite' -%}
-  {{ matrix_mx_puppet_twitter_database_engine }}://{{ matrix_mx_puppet_twitter_database_file }}
-  {%- endif -%}
+matrix_mx_puppet_twitter_database_db_name: matrix_mx_puppet_twitter
+
+matrix_mx_puppet_twitter_database_connection_string: 'postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_db_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml
index 663f822c498..3893981a565 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml
@@ -8,14 +8,6 @@
       The matrix-bridge-mx-puppet-twitter role needs to execute before the matrix-synapse role.
   when: "matrix_synapse_role_executed|default(False)"
 
-- name: Ensure MX Puppet Twitter image is pulled
-  docker_image:
-    name: "{{ matrix_mx_puppet_twitter_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_mx_puppet_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_twitter_docker_image_force_pull }}"
-  when: matrix_mx_puppet_twitter_enabled|bool and not matrix_mx_puppet_twitter_container_image_self_build
-
 - name: Ensure MX Puppet Twitter paths exist
   file:
     path: "{{ item.path }}"
@@ -30,6 +22,57 @@
     - { path: "{{ matrix_mx_puppet_twitter_docker_src_files_path }}", when: "{{ matrix_mx_puppet_twitter_container_image_self_build }}" }
   when: matrix_mx_puppet_twitter_enabled|bool and item.when|bool
 
+- name: Check if an old database file already exists
+  stat:
+    path: "{{ matrix_mx_puppet_twitter_base_path }}/database.db"
+  register: matrix_mx_puppet_twitter_stat_database
+
+- name: (Data relocation) Ensure matrix-mx-puppet-twitter.service is stopped
+  service:
+    name: matrix-mx-puppet-twitter
+    state: stopped
+    daemon_reload: yes
+  failed_when: false
+  when: "matrix_mx_puppet_twitter_stat_database.stat.exists"
+
+- name: (Data relocation) Move mx-puppet-twitter database file to ./data directory
+  command: "mv {{ matrix_mx_puppet_twitter_base_path }}/database.db {{ matrix_mx_puppet_twitter_data_path }}/database.db"
+  when: "matrix_mx_puppet_twitter_stat_database.stat.exists"
+
+- set_fact:
+    matrix_mx_puppet_twitter_requires_restart: false
+
+- block:
+    - name: Check if an SQLite database already exists
+      stat:
+        path: "{{ matrix_mx_puppet_twitter_sqlite_database_path_local }}"
+      register: matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result
+
+    - block:
+        - set_fact:
+            matrix_postgres_db_migration_request:
+              src: "{{ matrix_mx_puppet_twitter_sqlite_database_path_local }}"
+              dst: "{{ matrix_mx_puppet_twitter_database_connection_string }}"
+              caller: "{{ role_path|basename }}"
+              engine_variable_name: 'matrix_mx_puppet_twitter_database_engine'
+              engine_old: 'sqlite'
+              systemd_services_to_stop: ['matrix-mx-puppet-twitter.service']
+
+        - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
+
+        - set_fact:
+            matrix_mx_puppet_twitter_requires_restart: true
+      when: "matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_mx_puppet_twitter_database_engine == 'postgres'"
+
+- name: Ensure MX Puppet Twitter image is pulled
+  docker_image:
+    name: "{{ matrix_mx_puppet_twitter_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_mx_puppet_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_twitter_docker_image_force_pull }}"
+  when: matrix_mx_puppet_twitter_enabled|bool and not matrix_mx_puppet_twitter_container_image_self_build
+
 - name: Ensure MX Puppet Twitter repository is present on self build
   git:
     repo: "{{ matrix_mx_puppet_twitter_container_image_self_build_repo }}"
@@ -49,23 +92,6 @@
       pull: yes
   when: "matrix_mx_puppet_twitter_enabled|bool and matrix_mx_puppet_twitter_container_image_self_build"
 
-- name: Check if an old database file already exists
-  stat:
-    path: "{{ matrix_mx_puppet_twitter_base_path }}/database.db"
-  register: matrix_mx_puppet_twitter_stat_database
-
-- name: (Data relocation) Ensure matrix-mx-puppet-twitter.service is stopped
-  service:
-    name: matrix-mx-puppet-twitter
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
-  when: "matrix_mx_puppet_twitter_stat_database.stat.exists"
-
-- name: (Data relocation) Move mx-puppet-twitter database file to ./data directory
-  command: "mv {{ matrix_mx_puppet_twitter_base_path }}/database.db {{ matrix_mx_puppet_twitter_data_path }}/database.db"
-  when: "matrix_mx_puppet_twitter_stat_database.stat.exists"
-
 - name: Ensure mx-puppet-twitter config.yaml installed
   copy:
     content: "{{ matrix_mx_puppet_twitter_configuration|to_nice_yaml }}"
@@ -93,3 +119,9 @@
   service:
     daemon_reload: yes
   when: "matrix_mx_puppet_twitter_systemd_service_result.changed"
+
+- name: Ensure matrix-mx-puppet-twitter.service restarted, if necessary
+  service:
+    name: "matrix-mx-puppet-twitter.service"
+    state: restarted
+  when: "matrix_mx_puppet_twitter_requires_restart|bool"
diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
index 0b3bd8f82d7..d13a39e1e90 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/validate_config.yml
@@ -8,25 +8,3 @@
   with_items:
     - "matrix_mx_puppet_twitter_appservice_token"
     - "matrix_mx_puppet_twitter_homeserver_token"
-
-- block:
-  - name: Check if an SQLite database already exists
-    stat:
-      path: "{{ matrix_mx_puppet_twitter_sqlite_data_path }}/database.db"
-    register: matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result
-
-  - name: Fail if an SQLite database already exists when using Postgres
-    fail:
-      msg: >-
-        matrix_mx_puppet_twitter_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing SQLite database in {{ matrix_mx_puppet_twitter_data_path }}/database.db.
-        It appears that you've been using this bridge with the SQLite engine until now.
-        To continue using SQLite, opt into it explicitly: add `matrix_mx_puppet_twitter_database_engine: sqlite` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing SQLite database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the SQLite database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_mx_puppet_twitter_data_path }}/database.db postgres_connection_string_variable_name=matrix_mx_puppet_twitter_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result.stat.exists"
-  when: "matrix_mx_puppet_twitter_database_engine == 'postgres'"
-
-
diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
index cecb171d9e9..bdecf1dce07 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
+++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2
@@ -60,11 +60,11 @@ database:
   # Connection string to connect to the Postgres instance
   # with username "user", password "pass", host "localhost" and database name "dbname".
   # Modify each value as necessary
-  connString: {{ matrix_mx_puppet_twitter_database_connString | to_json }}
+  connString: {{ matrix_mx_puppet_twitter_database_connection_string|to_json }}
 {% else %}
   # Use SQLite3 as a database backend
   # The name of the database file
-  filename: {{ matrix_mx_puppet_twitter_database_file }}
+  filename: {{ matrix_mx_puppet_twitter_sqlite_database_path_in_container|to_json }}
 {% endif %}
 
 logging:

From 15f4cc924d30a62256cd30a0c73311223d74eac6 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 17:10:02 +0200
Subject: [PATCH 74/79] Rename variables (_database_db_name -> _database_name)

---
 group_vars/matrix_servers                     | 34 +++++++++----------
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 .../defaults/main.yml                         |  4 +--
 roles/matrix-dimension/defaults/main.yml      |  4 +--
 roles/matrix-ma1sd/defaults/main.yml          |  4 +--
 roles/matrix-ma1sd/templates/ma1sd.yaml.j2    |  2 +-
 roles/matrix-registration/defaults/main.yml   |  4 +--
 .../tasks/setup_install.yml                   |  2 +-
 18 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 283d5b6e3b1..21846592ecf 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -999,55 +999,55 @@ matrix_postgres_db_name: "homeserver"
 matrix_postgres_additional_databases: |
   {{
     ([{
-      'name': matrix_ma1sd_database_db_name,
+      'name': matrix_ma1sd_database_name,
       'username': matrix_ma1sd_database_username,
       'password': matrix_ma1sd_database_password,
     }] if (matrix_ma1sd_enabled and matrix_ma1sd_database_engine == 'postgres' and matrix_ma1sd_database_hostname == 'matrix-postgres') else [])
     +
     ([{
-      'name': matrix_bot_matrix_reminder_bot_database_db_name,
+      'name': matrix_bot_matrix_reminder_bot_database_name,
       'username': matrix_bot_matrix_reminder_bot_database_username,
       'password': matrix_bot_matrix_reminder_bot_database_password,
     }] if (matrix_bot_matrix_reminder_bot_enabled and matrix_bot_matrix_reminder_bot_database_engine == 'postgres' and matrix_bot_matrix_reminder_bot_database_hostname == 'matrix-postgres') else [])
     +
     ([{
-      'name': matrix_registration_database_db_name,
+      'name': matrix_registration_database_name,
       'username': matrix_registration_database_username,
       'password': matrix_registration_database_password,
     }] if (matrix_registration_enabled and matrix_registration_database_engine == 'postgres' and matrix_registration_database_hostname == 'matrix-postgres') else [])
     +
     ([{
-      'name': matrix_appservice_discord_database_db_name,
+      'name': matrix_appservice_discord_database_name,
       'username': matrix_appservice_discord_database_username,
       'password': matrix_appservice_discord_database_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_slack_database_db_name,
+      'name': matrix_appservice_slack_database_name,
       'username': matrix_appservice_slack_database_username,
       'password': matrix_appservice_slack_database_password,
     }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_appservice_irc_database_db_name,
+      'name': matrix_appservice_irc_database_name,
       'username': matrix_appservice_irc_database_username,
       'password': matrix_appservice_irc_database_password,
     }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_facebook_database_db_name,
+      'name': matrix_mautrix_facebook_database_name,
       'username': matrix_mautrix_facebook_database_username,
       'password': matrix_mautrix_facebook_database_password,
     }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_hangouts_database_db_name,
+      'name': matrix_mautrix_hangouts_database_name,
       'username': matrix_mautrix_hangouts_database_username,
       'password': matrix_mautrix_hangouts_database_password,
     }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_telegram_database_db_name,
+      'name': matrix_mautrix_telegram_database_name,
       'username': matrix_mautrix_telegram_database_username,
       'password': matrix_mautrix_telegram_database_password,
     }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mautrix_whatsapp_database_db_name,
+      'name': matrix_mautrix_whatsapp_database_name,
       'username': matrix_mautrix_whatsapp_database_username,
       'password': matrix_mautrix_whatsapp_database_password,
     }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
@@ -1057,36 +1057,36 @@ matrix_postgres_additional_databases: |
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
     + ([{
-      'name': matrix_mx_puppet_skype_database_db_name,
+      'name': matrix_mx_puppet_skype_database_name,
       'username': matrix_mx_puppet_skype_database_username,
       'password': matrix_mx_puppet_skype_database_password,
     }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
     + ([{
-      'name': matrix_mx_puppet_slack_database_db_name,
+      'name': matrix_mx_puppet_slack_database_name,
       'username': matrix_mx_puppet_slack_database_username,
       'password': matrix_mx_puppet_slack_database_password,
     }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_twitter_database_db_name,
+      'name': matrix_mx_puppet_twitter_database_name,
       'username': matrix_mx_puppet_twitter_database_username,
       'password': matrix_mx_puppet_twitter_database_password,
     }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_instagram_database_db_name,
+      'name': matrix_mx_puppet_instagram_database_name,
       'username': matrix_mx_puppet_instagram_database_username,
       'password': matrix_mx_puppet_instagram_database_password,
     }] if (matrix_mx_puppet_instagram_enabled
            and matrix_mx_puppet_instagram_database_engine == 'postgres'
            and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_discord_database_db_name,
+      'name': matrix_mx_puppet_discord_database_name,
       'username': matrix_mx_puppet_discord_database_username,
       'password': matrix_mx_puppet_discord_database_password,
     }] if (matrix_mx_puppet_discord_enabled
            and matrix_mx_puppet_discord_database_engine == 'postgres'
            and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
      + ([{
-      'name': matrix_mx_puppet_steam_database_db_name,
+      'name': matrix_mx_puppet_steam_database_name,
       'username': matrix_mx_puppet_steam_database_username,
       'password': matrix_mx_puppet_steam_database_password,
     }] if (matrix_mx_puppet_steam_enabled
@@ -1094,7 +1094,7 @@ matrix_postgres_additional_databases: |
            and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
      +
     ([{
-      'name': matrix_dimension_database_db_name,
+      'name': matrix_dimension_database_name,
       'username': matrix_dimension_database_username,
       'password': matrix_dimension_database_password,
     }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else [])
diff --git a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
index 70e7b2d2fb8..29bc83073a0 100644
--- a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
+++ b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml
@@ -37,9 +37,9 @@ matrix_bot_matrix_reminder_bot_database_username: 'matrix_reminder_bot'
 matrix_bot_matrix_reminder_bot_database_password: 'some-password'
 matrix_bot_matrix_reminder_bot_database_hostname: 'matrix-postgres'
 matrix_bot_matrix_reminder_bot_database_port: 5432
-matrix_bot_matrix_reminder_bot_database_db_name: 'matrix_reminder_bot'
+matrix_bot_matrix_reminder_bot_database_name: 'matrix_reminder_bot'
 
-matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_db_name }}'
+matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_name }}'
 
 matrix_bot_matrix_reminder_bot_storage_database: "{{
 	{
diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml
index 8e453a15897..c7cdddb6e50 100644
--- a/roles/matrix-bridge-appservice-discord/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml
@@ -57,12 +57,12 @@ matrix_appservice_discord_database_username: 'matrix_appservice_discord'
 matrix_appservice_discord_database_password: 'some-password'
 matrix_appservice_discord_database_hostname: 'matrix-postgres'
 matrix_appservice_discord_database_port: 5432
-matrix_appservice_discord_database_db_name: 'matrix_appservice_discord'
+matrix_appservice_discord_database_name: 'matrix_appservice_discord'
 
 # These 2 variables are what actually ends up in the bridge configuration.
 # It's best if you don't change them directly, but rather redefine the sub-variables that constitute them.
 matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}"
-matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_db_name }}'
+matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_name }}'
 
 
 # Tells whether the bot should make use of "Privileged Gateway Intents".
diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
index c02197a5b37..580934dba42 100644
--- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml
@@ -49,9 +49,9 @@ matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook'
 matrix_mautrix_facebook_database_password: 'some-password'
 matrix_mautrix_facebook_database_hostname: 'matrix-postgres'
 matrix_mautrix_facebook_database_port: 5432
-matrix_mautrix_facebook_database_db_name: 'matrix_mautrix_facebook'
+matrix_mautrix_facebook_database_name: 'matrix_mautrix_facebook'
 
-matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_db_name }}'
+matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_name }}'
 
 matrix_mautrix_facebook_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
index 5d38289ba1c..984bec4862d 100644
--- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml
@@ -56,9 +56,9 @@ matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts'
 matrix_mautrix_hangouts_database_password: 'some-password'
 matrix_mautrix_hangouts_database_hostname: 'matrix-postgres'
 matrix_mautrix_hangouts_database_port: 5432
-matrix_mautrix_hangouts_database_db_name: 'matrix_mautrix_hangouts'
+matrix_mautrix_hangouts_database_name: 'matrix_mautrix_hangouts'
 
-matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_db_name }}'
+matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_name }}'
 
 matrix_mautrix_hangouts_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
index fcdf95db295..ceebd3ec13c 100644
--- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml
@@ -60,9 +60,9 @@ matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram'
 matrix_mautrix_telegram_database_password: 'some-password'
 matrix_mautrix_telegram_database_hostname: 'matrix-postgres'
 matrix_mautrix_telegram_database_port: 5432
-matrix_mautrix_telegram_database_db_name: 'matrix_mautrix_telegram'
+matrix_mautrix_telegram_database_name: 'matrix_mautrix_telegram'
 
-matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_db_name }}'
+matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_name }}'
 
 matrix_mautrix_telegram_appservice_database: "{{
 	{
diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
index 7e198b36883..0d4f9852774 100644
--- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
+++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml
@@ -44,9 +44,9 @@ matrix_mautrix_whatsapp_database_username: 'matrix_mautrix_whatsapp'
 matrix_mautrix_whatsapp_database_password: 'some-password'
 matrix_mautrix_whatsapp_database_hostname: 'matrix-postgres'
 matrix_mautrix_whatsapp_database_port: 5432
-matrix_mautrix_whatsapp_database_db_name: 'matrix_mautrix_whatsapp'
+matrix_mautrix_whatsapp_database_name: 'matrix_mautrix_whatsapp'
 
-matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_db_name }}'
+matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_name }}'
 
 matrix_mautrix_whatsapp_appservice_database_type: "{{
 	{
diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
index ad86a2416a9..97b203137ee 100644
--- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml
@@ -65,9 +65,9 @@ matrix_mx_puppet_discord_database_username: matrix_mx_puppet_discord
 matrix_mx_puppet_discord_database_password: ~
 matrix_mx_puppet_discord_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_discord_database_port: 5432
-matrix_mx_puppet_discord_database_db_name: matrix_mx_puppet_discord
+matrix_mx_puppet_discord_database_name: matrix_mx_puppet_discord
 
-matrix_mx_puppet_discord_database_connection_string: 'postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_discord_database_connection_string: 'postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
index 61bf3dcfeac..cd08c010a27 100644
--- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml
@@ -55,9 +55,9 @@ matrix_mx_puppet_instagram_database_username: matrix_mx_puppet_instagram
 matrix_mx_puppet_instagram_database_password: ~
 matrix_mx_puppet_instagram_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_instagram_database_port: 5432
-matrix_mx_puppet_instagram_database_db_name: matrix_mx_puppet_instagram
+matrix_mx_puppet_instagram_database_name: matrix_mx_puppet_instagram
 
-matrix_mx_puppet_instagram_database_connection_string: 'postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_instagram_database_connection_string: 'postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
index f3e17a593a2..83cd3dc55b4 100644
--- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml
@@ -63,9 +63,9 @@ matrix_mx_puppet_skype_database_username: matrix_mx_puppet_skype
 matrix_mx_puppet_skype_database_password: ~
 matrix_mx_puppet_skype_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_skype_database_port: 5432
-matrix_mx_puppet_skype_database_db_name: matrix_mx_puppet_skype
+matrix_mx_puppet_skype_database_name: matrix_mx_puppet_skype
 
-matrix_mx_puppet_skype_database_connection_string: 'postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_skype_database_connection_string: 'postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
index 9aad602a4ba..70b98eceebb 100644
--- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml
@@ -67,9 +67,9 @@ matrix_mx_puppet_slack_database_username: matrix_mx_puppet_slack
 matrix_mx_puppet_slack_database_password: ~
 matrix_mx_puppet_slack_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_slack_database_port: 5432
-matrix_mx_puppet_slack_database_db_name: matrix_mx_puppet_slack
+matrix_mx_puppet_slack_database_name: matrix_mx_puppet_slack
 
-matrix_mx_puppet_slack_database_connection_string: 'postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_slack_database_connection_string: 'postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
index c1f6a385584..15fa889f0e9 100644
--- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml
@@ -64,9 +64,9 @@ matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam
 matrix_mx_puppet_steam_database_password: ~
 matrix_mx_puppet_steam_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_steam_database_port: 5432
-matrix_mx_puppet_steam_database_db_name: matrix_mx_puppet_steam
+matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam
 
-matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
index da7a9ae7f48..28639fdadb4 100644
--- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
+++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml
@@ -70,9 +70,9 @@ matrix_mx_puppet_twitter_database_username: mx_puppet_twitter
 matrix_mx_puppet_twitter_database_password: ~
 matrix_mx_puppet_twitter_database_hostname: 'matrix-postgres'
 matrix_mx_puppet_twitter_database_port: 5432
-matrix_mx_puppet_twitter_database_db_name: matrix_mx_puppet_twitter
+matrix_mx_puppet_twitter_database_name: matrix_mx_puppet_twitter
 
-matrix_mx_puppet_twitter_database_connection_string: 'postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_db_name }}?sslmode=disable'
+matrix_mx_puppet_twitter_database_connection_string: 'postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable'
 
 # Default configuration template which covers the generic use case.
 # You can customize it by controlling the various variables inside it.
diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml
index a6a3bf5d40d..8a691946f73 100644
--- a/roles/matrix-dimension/defaults/main.yml
+++ b/roles/matrix-dimension/defaults/main.yml
@@ -57,9 +57,9 @@ matrix_dimension_database_username: 'matrix_dimension'
 matrix_dimension_database_password: 'some-password'
 matrix_dimension_database_hostname: 'matrix-postgres'
 matrix_dimension_database_port: 5432
-matrix_dimension_database_db_name: 'matrix_dimension'
+matrix_dimension_database_name: 'matrix_dimension'
 
-matrix_dimension_database_connection_string: 'postgres://{{ matrix_dimension_database_username }}:{{ matrix_dimension_database_password }}@{{ matrix_dimension_database_hostname }}:{{ matrix_dimension_database_port }}/{{ matrix_dimension_database_db_name }}'
+matrix_dimension_database_connection_string: 'postgres://{{ matrix_dimension_database_username }}:{{ matrix_dimension_database_password }}@{{ matrix_dimension_database_hostname }}:{{ matrix_dimension_database_port }}/{{ matrix_dimension_database_name }}'
 
 
 # Default Dimension configuration template which covers the generic use case.
diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml
index 42e36e924fe..2932f3ed422 100644
--- a/roles/matrix-ma1sd/defaults/main.yml
+++ b/roles/matrix-ma1sd/defaults/main.yml
@@ -56,9 +56,9 @@ matrix_ma1sd_database_username: 'matrix_ma1sd'
 matrix_ma1sd_database_password: 'some-password'
 matrix_ma1sd_database_hostname: 'matrix-postgres'
 matrix_ma1sd_database_port: 5432
-matrix_ma1sd_database_db_name: 'matrix_ma1sd'
+matrix_ma1sd_database_name: 'matrix_ma1sd'
 
-matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}'
+matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_name }}'
 
 
 # ma1sd has serveral supported identity stores.
diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
index 43b4022dca9..9a426c477f1 100644
--- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
+++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2
@@ -20,7 +20,7 @@ storage:
     backend: postgresql
     provider:
       postgresql:
-        database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_db_name }}
+        database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_name }}
         username: {{ matrix_ma1sd_database_username|to_json }}
         password: {{ matrix_ma1sd_database_password|to_json }}
   {% endif %}
diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml
index d85faf89a9d..065e0c482a4 100644
--- a/roles/matrix-registration/defaults/main.yml
+++ b/roles/matrix-registration/defaults/main.yml
@@ -48,9 +48,9 @@ matrix_registration_database_username: 'matrix_registration'
 matrix_registration_database_password: 'some-password'
 matrix_registration_database_hostname: 'matrix-postgres'
 matrix_registration_database_port: 5432
-matrix_registration_database_db_name: 'matrix_registration'
+matrix_registration_database_name: 'matrix_registration'
 
-matrix_registration_database_connection_string: 'postgresql://{{ matrix_registration_database_username }}:{{ matrix_registration_database_password }}@{{ matrix_registration_database_hostname }}:{{ matrix_registration_database_port }}/{{ matrix_registration_database_db_name }}'
+matrix_registration_database_connection_string: 'postgresql://{{ matrix_registration_database_username }}:{{ matrix_registration_database_password }}@{{ matrix_registration_database_hostname }}:{{ matrix_registration_database_port }}/{{ matrix_registration_database_name }}'
 
 # For some reason, matrix-registraiton expects the `db` field to be like this: `sqlite:////data/db.sqlite3`.
 # (seems like one too many slashes, but..)
diff --git a/roles/matrix-registration/tasks/setup_install.yml b/roles/matrix-registration/tasks/setup_install.yml
index 708cb1df561..9b6d126018d 100644
--- a/roles/matrix-registration/tasks/setup_install.yml
+++ b/roles/matrix-registration/tasks/setup_install.yml
@@ -22,7 +22,7 @@
               # which makes matrix-registration choke on it later on when comparing dates.
               additional_psql_statements_list:
                 - ALTER TABLE tokens ALTER COLUMN ex_date TYPE TIMESTAMP WITHOUT TIME ZONE;
-              additional_psql_statements_db_name: "{{ matrix_registration_database_db_name }}"
+              additional_psql_statements_db_name: "{{ matrix_registration_database_name }}"
 
         - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml"
 

From 815a80c290d2527a74a26acb47e6f1fbbb3f000b Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 17:16:56 +0200
Subject: [PATCH 75/79] Fix alignment issues

---
 group_vars/matrix_servers | 53 ++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 23 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 21846592ecf..318ebac777c 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -1021,78 +1021,85 @@ matrix_postgres_additional_databases: |
       'username': matrix_appservice_discord_database_username,
       'password': matrix_appservice_discord_database_password,
     }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_appservice_slack_database_name,
       'username': matrix_appservice_slack_database_username,
       'password': matrix_appservice_slack_database_password,
     }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_appservice_irc_database_name,
       'username': matrix_appservice_irc_database_username,
       'password': matrix_appservice_irc_database_password,
     }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mautrix_facebook_database_name,
       'username': matrix_mautrix_facebook_database_username,
       'password': matrix_mautrix_facebook_database_password,
     }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mautrix_hangouts_database_name,
       'username': matrix_mautrix_hangouts_database_username,
       'password': matrix_mautrix_hangouts_database_password,
     }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mautrix_telegram_database_name,
       'username': matrix_mautrix_telegram_database_username,
       'password': matrix_mautrix_telegram_database_password,
     }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mautrix_whatsapp_database_name,
       'username': matrix_mautrix_whatsapp_database_username,
       'password': matrix_mautrix_whatsapp_database_password,
     }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': 'matrix_bridge_sms',
       'username': 'matrix_bridge_sms',
       'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid,
     }] if matrix_sms_bridge_enabled else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mx_puppet_skype_database_name,
       'username': matrix_mx_puppet_skype_database_username,
       'password': matrix_mx_puppet_skype_database_password,
     }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else [])
-    + ([{
+    +
+    ([{
       'name': matrix_mx_puppet_slack_database_name,
       'username': matrix_mx_puppet_slack_database_username,
       'password': matrix_mx_puppet_slack_database_password,
     }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else [])
-     + ([{
+    +
+    ([{
       'name': matrix_mx_puppet_twitter_database_name,
       'username': matrix_mx_puppet_twitter_database_username,
       'password': matrix_mx_puppet_twitter_database_password,
     }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else [])
-     + ([{
+    +
+    ([{
       'name': matrix_mx_puppet_instagram_database_name,
       'username': matrix_mx_puppet_instagram_database_username,
       'password': matrix_mx_puppet_instagram_database_password,
-    }] if (matrix_mx_puppet_instagram_enabled
-           and matrix_mx_puppet_instagram_database_engine == 'postgres'
-           and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
-     + ([{
+    }] if (matrix_mx_puppet_instagram_enabled and matrix_mx_puppet_instagram_database_engine == 'postgres' and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else [])
+    +
+    ([{
       'name': matrix_mx_puppet_discord_database_name,
       'username': matrix_mx_puppet_discord_database_username,
       'password': matrix_mx_puppet_discord_database_password,
-    }] if (matrix_mx_puppet_discord_enabled
-           and matrix_mx_puppet_discord_database_engine == 'postgres'
-           and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
-     + ([{
+    }] if (matrix_mx_puppet_discord_enabled  and matrix_mx_puppet_discord_database_engine == 'postgres' and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else [])
+    +
+    ([{
       'name': matrix_mx_puppet_steam_database_name,
       'username': matrix_mx_puppet_steam_database_username,
       'password': matrix_mx_puppet_steam_database_password,
-    }] if (matrix_mx_puppet_steam_enabled
-           and matrix_mx_puppet_steam_database_engine == 'postgres'
-           and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
-     +
+    }] if (matrix_mx_puppet_steam_enabled and matrix_mx_puppet_steam_database_engine == 'postgres' and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else [])
+    +
     ([{
       'name': matrix_dimension_database_name,
       'username': matrix_dimension_database_username,

From 715bdf2c6487c56be5ad17a4e7b80d94efa46ce2 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 19:32:43 +0200
Subject: [PATCH 76/79] Add support for automatic (nedb -> Postgres) migration
 to mx-appservice-irc

---
 group_vars/matrix_servers                     |  3 +-
 roles/matrix-base/defaults/main.yml           |  1 +
 .../defaults/main.yml                         | 17 +++--
 .../tasks/migrate_nedb_to_postgres.yml        | 53 ++++++++++++++++
 .../tasks/setup_install.yml                   | 63 ++++++++++++-------
 .../tasks/validate_config.yml                 | 20 ------
 .../templates/config.yaml.j2                  |  4 +-
 roles/matrix-postgres/tasks/import_nedb.yml   | 21 +------
 roles/matrix-postgres/tasks/main.yml          |  8 ---
 9 files changed, 109 insertions(+), 81 deletions(-)
 create mode 100644 roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 318ebac777c..4ddadb43229 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -181,8 +181,7 @@ matrix_appservice_irc_appservice_token: "{{ matrix_synapse_macaroon_secret_key |
 
 matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'irc.hs.token') | to_uuid }}"
 
-# Postgres is the default, except if not using `matrix_postgres` (internal postgres)
-matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}"
+matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'nedb' }}"
 matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}"
 
 
diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml
index e0522ba8fb0..d0fdcdf8666 100644
--- a/roles/matrix-base/defaults/main.yml
+++ b/roles/matrix-base/defaults/main.yml
@@ -106,6 +106,7 @@ matrix_docker_package_name: docker-ce
 run_postgres_import: true
 run_postgres_upgrade: true
 run_postgres_import_sqlite_db: true
+run_postgres_import_nedb: true
 run_postgres_vacuum: true
 run_synapse_register_user: true
 run_synapse_update_user_password: true
diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml
index 5110fd852c0..0b671e76299 100644
--- a/roles/matrix-bridge-appservice-irc/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml
@@ -23,12 +23,17 @@ matrix_appservice_irc_database_hostname: 'matrix-postgres'
 matrix_appservice_irc_database_port: 5432
 matrix_appservice_irc_database_name: matrix_appservice_irc
 
-matrix_appservice_irc_database_connString: >-2
-  {%- if matrix_appservice_irc_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable
-  {%- elif matrix_appservice_irc_database_engine == 'nedb' -%}
-  {{ matrix_appservice_irc_database_engine }}://{{ matrix_appservice_irc_database_file }}
-  {%- endif -%}
+# This is just the Postgres connection string, if Postgres is used.
+# Naming clashes with `matrix_appservice_irc_database_connectionString` somewhat.
+matrix_appservice_irc_database_connection_string: 'postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable'
+
+# This is what actually goes into `database.connectionString` for the bridge.
+matrix_appservice_irc_database_connectionString: "{{
+	{
+		'nedb': 'nedb:///data',
+		'postgres': matrix_appservice_irc_database_connection_string,
+	}[matrix_appservice_irc_database_engine]
+}}"
 
 matrix_appservice_irc_ircService_servers: []
 
diff --git a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml
new file mode 100644
index 00000000000..bc6525ec158
--- /dev/null
+++ b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml
@@ -0,0 +1,53 @@
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate."
+  when: "not matrix_postgres_enabled|bool"
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+- name: Ensure matrix-appservice-irc is stopped
+  service:
+    name: matrix-appservice-irc
+    state: stopped
+
+- name: Import appservice-irc NeDB database into Postgres
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_irc_docker_image }}
+      -c
+      '/usr/local/bin/node /app/lib/scripts/migrate-db-to-pgres.js --dbdir /data --privateKey /data/passkey.pem --connectionString {{ matrix_appservice_irc_database_connection_string }}'
+
+- name: Archive NeDB database files
+  command:
+    cmd: "mv {{ matrix_appservice_irc_data_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}.backup"
+  with_items:
+    - rooms.db
+    - users.db
diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
index 5e313347fce..a748df96226 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
@@ -1,12 +1,5 @@
 ---
 
-- name: Ensure Appservice IRC image is pulled
-  docker_image:
-    name: "{{ matrix_appservice_irc_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}"
-
 - name: Ensure Appservice IRC paths exist
   file:
     path: "{{ item }}"
@@ -24,25 +17,49 @@
     path: "{{ matrix_appservice_irc_base_path }}/passkey.pem"
   register: matrix_appservice_irc_stat_passkey
 
-- name: (Data relocation) Ensure matrix-appservice-irc.service is stopped
-  service:
-    name: matrix-appservice-irc
-    state: stopped
-    daemon_reload: yes
-  failed_when: false
+- block:
+    - name: (Data relocation) Ensure matrix-appservice-irc.service is stopped
+      service:
+        name: matrix-appservice-irc
+        state: stopped
+        daemon_reload: yes
+      failed_when: false
+
+    - name: (Data relocation) Move AppService IRC passkey.pem file to ./data directory
+      command: "mv {{ matrix_appservice_irc_base_path }}/passkey.pem {{ matrix_appservice_irc_data_path }}/passkey.pem"
+
+    - name: (Data relocation) Move AppService IRC database files to ./data directory
+      command: "mv {{ matrix_appservice_irc_base_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}"
+      with_items:
+        - rooms.db
+        - users.db
+      failed_when: false
   when: "matrix_appservice_irc_stat_passkey.stat.exists"
 
-- name: (Data relocation) Move AppService IRC passkey.pem file to ./data directory
-  command: "mv {{ matrix_appservice_irc_base_path }}/passkey.pem {{ matrix_appservice_irc_data_path }}/passkey.pem"
-  when: "matrix_appservice_irc_stat_passkey.stat.exists"
 
-- name: (Data relocation) Move AppService IRC database files to ./data directory
-  command: "mv {{ matrix_appservice_irc_base_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}"
-  with_items:
-    - rooms.db
-    - users.db
-  failed_when: false
-  when: "matrix_appservice_irc_stat_passkey.stat.exists"
+- set_fact:
+    matrix_appservice_irc_requires_restart: false
+
+- block:
+    - name: Check if a nedb database already exists
+      stat:
+        path: "{{ matrix_appservice_irc_data_path }}/users.db"
+      register: matrix_appservice_irc_nedb_database_path_local_stat_result
+
+    - block:
+        - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml"
+
+        - set_fact:
+            matrix_appservice_irc_requires_restart: true
+      when: "matrix_appservice_irc_nedb_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_appservice_irc_database_engine == 'postgres'"
+
+- name: Ensure Appservice IRC image is pulled
+  docker_image:
+    name: "{{ matrix_appservice_irc_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}"
 
 - name: Ensure Matrix Appservice IRC config installed
   copy:
diff --git a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
index 9b89a3400f6..bd08427c2d0 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/validate_config.yml
@@ -33,23 +33,3 @@
   when: "item.old in vars"
   with_items:
     - {'old': 'matrix_appservice_irc_container_expose_client_server_api_port', 'new': '<superseded by matrix_appservice_irc_container_http_host_bind_port>'}
-
-- block:
-  - name: Check if a neDB database already exists
-    stat:
-      path: "{{ matrix_appservice_irc_data_path }}/"
-    register: matrix_appservice_irc_nedb_stat_result
-
-  - name: Fail if an neDB database already exists when using Postgres
-    fail:
-      msg: >-2
-        matrix_appservice_irc_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing neDB database in {{ matrix_appservice_irc_data_path }}/.
-        It appears that you've been using this bridge with the neDB engine until now.
-        To continue using neDB, opt into it explicitly: add `matrix_appservice_irc_database_engine: nedb` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing neDB database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-irc-nedb --extra-vars='nedb_database_path={{ matrix_appservice_irc_data_path }} postgres_connection_string_variable_name=matrix_appservice_irc_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_appservice_irc_nedb_stat_result.stat.exists"
-  when: "matrix_appservice_irc_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
index 0da28403f91..94bbda7b812 100644
--- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2
@@ -127,8 +127,8 @@ advanced:
 # Use an external database to store bridge state.
 database:
   # database engine (must be 'postgres' or 'nedb'). Default: nedb
-  engine: {{ matrix_appservice_irc_database_engine }}
+  engine: {{ matrix_appservice_irc_database_engine|to_json }}
   # Either a PostgreSQL connection string, or a path to the NeDB storage directory.
   # For postgres, it must start with postgres://
   # For NeDB, it must start with nedb://. The path is relative to the project directory.
-  connectionString: {{ matrix_appservice_irc_database_connString | to_json }}
+  connectionString: {{ matrix_appservice_irc_database_connectionString|to_json }}
diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml
index cc1f9d78815..2a3dd587898 100644
--- a/roles/matrix-postgres/tasks/import_nedb.yml
+++ b/roles/matrix-postgres/tasks/import_nedb.yml
@@ -67,22 +67,6 @@
   become: false
   when: "matrix_postgres_service_start_result.changed|bool"
 
-# See https://github.com/matrix-org/matrix-appservice-irc/wiki/Migrating-from-NEdB-to-PostgreSQL
-- name: Import appservice_irc NeDB database from {{ sqlite_database_path }} into Postgres
-  when: database == 'appservice_irc'
-  command:
-    cmd: >-
-      {{ matrix_host_command_docker }} run
-      --rm
-      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
-      --cap-drop=ALL
-      --network={{ matrix_docker_network }}
-      --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro
-      --entrypoint=/bin/sh
-      {{ matrix_appservice_irc_docker_image }}
-      -c
-      './scripts/migrate-db-to-pgres.sh -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
-
 # No migration.sh available, but found this:
 # https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts
 # Usage should be similar to appservice_irc
@@ -95,15 +79,12 @@
       --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
       --cap-drop=ALL
       --network={{ matrix_docker_network }}
-      --mount type=bind,src={{ matrix_appservice_irc_data_path }}:/data:ro
+      --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data,ro
       --entrypoint=/bin/sh
       {{ matrix_appservice_slack_docker_image }}
       -c
       'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
 
-- name: Archive NeDB database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup)
-  command:
-    cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup"
 
 - name: Inject result
   set_fact:
diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml
index 5180131478d..86b8f8721e2 100644
--- a/roles/matrix-postgres/tasks/main.yml
+++ b/roles/matrix-postgres/tasks/main.yml
@@ -32,14 +32,6 @@
   tags:
     - import-generic-sqlite-db
 
-# Imports appservice-irc NeDB into postgres
-- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
-  vars:
-    database: appservice_irc
-  when: run_postgres_import_nedb|bool
-  tags:
-    - import-irc-nedb
-
 # Imports slacks neDB to postgres.
 - import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
   vars:

From 9b95e1937c61a2b21d3d11b3d497e16c6176d593 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 19:34:08 +0200
Subject: [PATCH 77/79] Auto-restart matrix-appservice-irc after (nedb ->
 Postgres) migration

---
 roles/matrix-bridge-appservice-irc/tasks/setup_install.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
index a748df96226..00568c0df7b 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml
@@ -36,7 +36,6 @@
       failed_when: false
   when: "matrix_appservice_irc_stat_passkey.stat.exists"
 
-
 - set_fact:
     matrix_appservice_irc_requires_restart: false
 
@@ -164,3 +163,9 @@
   service:
     daemon_reload: yes
   when: "matrix_appservice_irc_systemd_service_result.changed"
+
+- name: Ensure matrix-appservice-irc.service restarted, if necessary
+  service:
+    name: "matrix-appservice-irc.service"
+    state: restarted
+  when: "matrix_appservice_irc_requires_restart|bool"

From 8675dedbdb310425d309f6bf1725d7a6c415279e Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Tue, 22 Dec 2020 19:56:52 +0200
Subject: [PATCH 78/79] Add support for automatic (nedb -> Postgres) migration
 to matrix-appservice-slack

---
 roles/matrix-base/defaults/main.yml           |  1 -
 .../tasks/migrate_nedb_to_postgres.yml        | 11 +++
 .../defaults/main.yml                         | 19 ++--
 .../tasks/migrate_nedb_to_postgres.yml        | 66 +++++++++++++
 .../tasks/setup_install.yml                   | 37 +++++--
 .../tasks/validate_config.yml                 | 20 ----
 .../templates/config.yaml.j2                  |  7 +-
 roles/matrix-postgres/tasks/import_nedb.yml   | 98 -------------------
 roles/matrix-postgres/tasks/main.yml          |  8 --
 9 files changed, 123 insertions(+), 144 deletions(-)
 create mode 100644 roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml
 delete mode 100644 roles/matrix-postgres/tasks/import_nedb.yml

diff --git a/roles/matrix-base/defaults/main.yml b/roles/matrix-base/defaults/main.yml
index d0fdcdf8666..e0522ba8fb0 100644
--- a/roles/matrix-base/defaults/main.yml
+++ b/roles/matrix-base/defaults/main.yml
@@ -106,7 +106,6 @@ matrix_docker_package_name: docker-ce
 run_postgres_import: true
 run_postgres_upgrade: true
 run_postgres_import_sqlite_db: true
-run_postgres_import_nedb: true
 run_postgres_vacuum: true
 run_synapse_register_user: true
 run_synapse_update_user_password: true
diff --git a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml
index bc6525ec158..3fab195ab0a 100644
--- a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml
+++ b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml
@@ -51,3 +51,14 @@
   with_items:
     - rooms.db
     - users.db
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your appservice-irc database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_irc_data_path }}/*.db` to `{{ matrix_appservice_irc_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files."
+        ]
+      }}
diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml
index 365493ff21a..28646a3c4a8 100644
--- a/roles/matrix-bridge-appservice-slack/defaults/main.yml
+++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml
@@ -51,13 +51,18 @@ matrix_appservice_slack_database_password: ~
 matrix_appservice_slack_database_hostname: 'matrix-postgres'
 matrix_appservice_slack_database_port: 5432
 matrix_appservice_slack_database_name: matrix_appservice_slack
-matrix_appservice_slack_database_file: /data
-matrix_appservice_slack_database_connString: >-2
-  {%- if matrix_appservice_slack_database_engine == 'postgres' -%}
-  postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable
-  {%- elif matrix_appservice_slack_database_engine == 'nedb' -%}
-  {{ matrix_appservice_slack_database_engine }}://{{ matrix_appservice_slack_database_file }}
-  {%- endif -%}
+
+# This is just the Postgres connection string, if Postgres is used.
+# Naming clashes with `matrix_appservice_slack_database_connectionString` somewhat.
+matrix_appservice_slack_database_connection_string: 'postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable'
+
+# This is what actually goes into `database.connectionString` for the bridge.
+matrix_appservice_slack_database_connectionString: "{{
+	{
+		'nedb': 'nedb:///data',
+		'postgres': matrix_appservice_slack_database_connection_string,
+	}[matrix_appservice_slack_database_engine]
+}}"
 
 
 matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}"
diff --git a/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml
new file mode 100644
index 00000000000..fedad977571
--- /dev/null
+++ b/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml
@@ -0,0 +1,66 @@
+- name: Fail if Postgres not enabled
+  fail:
+    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate."
+  when: "not matrix_postgres_enabled|bool"
+
+# Defaults
+
+- name: Set postgres_start_wait_time, if not provided
+  set_fact:
+    postgres_start_wait_time: 15
+  when: "postgres_start_wait_time|default('') == ''"
+
+# Actual import work
+
+- name: Ensure matrix-postgres is started
+  service:
+    name: matrix-postgres
+    state: started
+    daemon_reload: yes
+  register: matrix_postgres_service_start_result
+
+- name: Wait a bit, so that Postgres can start
+  wait_for:
+    timeout: "{{ postgres_start_wait_time }}"
+  delegate_to: 127.0.0.1
+  become: false
+  when: "matrix_postgres_service_start_result.changed|bool"
+
+- name: Ensure matrix-appservice-slack is stopped
+  service:
+    name: matrix-appservice-slack
+    state: stopped
+
+- name: Import appservice-slack NeDB database into Postgres
+  command:
+    cmd: >-
+      {{ matrix_host_command_docker }} run
+      --rm
+      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
+      --cap-drop=ALL
+      --network={{ matrix_docker_network }}
+      --mount type=bind,src={{ matrix_appservice_slack_data_path }},dst=/data
+      --entrypoint=/bin/sh
+      {{ matrix_appservice_slack_docker_image }}
+      -c
+      '/usr/local/bin/node /usr/src/app/lib/scripts/migrateToPostgres.js --dbdir /data --connectionString {{ matrix_appservice_slack_database_connection_string }}'
+
+- name: Archive NeDB database files
+  command:
+    cmd: "mv {{ matrix_appservice_slack_data_path }}/{{ item }} {{ matrix_appservice_slack_data_path }}/{{ item }}.backup"
+  with_items:
+    - teams.db
+    - room-store.db
+    - user-store.db
+    - event-store.db
+
+- name: Inject result
+  set_fact:
+    matrix_playbook_runtime_results: |
+      {{
+        matrix_playbook_runtime_results|default([])
+        +
+        [
+          "NOTE: Your appservice-slack database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_slack_data_path }}/*.db` to `{{ matrix_appservice_slack_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files."
+        ]
+      }}
diff --git a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml
index 94e0fedf92a..29b0f39e9bf 100644
--- a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml
+++ b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml
@@ -1,12 +1,5 @@
 ---
 
-- name: Ensure Appservice Slack image is pulled
-  docker_image:
-    name: "{{ matrix_appservice_slack_docker_image }}"
-    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
-    force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
-    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}"
-
 - name: Ensure AppService Slack paths exist
   file:
     path: "{{ item }}"
@@ -19,6 +12,30 @@
     - "{{ matrix_appservice_slack_config_path }}"
     - "{{ matrix_appservice_slack_data_path }}"
 
+- set_fact:
+    matrix_appservice_slack_requires_restart: false
+
+- block:
+    - name: Check if a nedb database already exists
+      stat:
+        path: "{{ matrix_appservice_slack_data_path }}/teams.db"
+      register: matrix_appservice_slack_nedb_database_path_local_stat_result
+
+    - block:
+        - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml"
+
+        - set_fact:
+            matrix_appservice_slack_requires_restart: true
+      when: "matrix_appservice_slack_nedb_database_path_local_stat_result.stat.exists|bool"
+  when: "matrix_appservice_slack_database_engine == 'postgres'"
+
+- name: Ensure Appservice Slack image is pulled
+  docker_image:
+    name: "{{ matrix_appservice_slack_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}"
+
 - name: Ensure Matrix Appservice Slack config installed
   copy:
     content: "{{ matrix_appservice_slack_configuration|to_nice_yaml }}"
@@ -46,3 +63,9 @@
   service:
     daemon_reload: yes
   when: "matrix_appservice_slack_systemd_service_result.changed"
+
+- name: Ensure matrix-appservice-slack.service restarted, if necessary
+  service:
+    name: "matrix-appservice-slack.service"
+    state: restarted
+  when: "matrix_appservice_slack_requires_restart|bool"
diff --git a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
index 5421b112a8d..8af10f2f54c 100644
--- a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
+++ b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml
@@ -10,23 +10,3 @@
     - "matrix_appservice_slack_appservice_token"
     - "matrix_appservice_slack_homeserver_token"
     - "matrix_appservice_slack_id_token"
-
-- block:
-  - name: Check if a neDB database already exists
-    stat:
-      path: "{{ matrix_appservice_slack_data_path }}/"
-    register: matrix_appservice_slack_nedb_stat_result
-
-  - name: Fail if an neDB database already exists when using Postgres
-    fail:
-      msg: >-2
-        matrix_appservice_slack_database_engine has been set to `postgres` (which is our new default now).
-        However, we've discovered an existing neDB database in {{ matrix_appservice_slack_data_path }}/.
-        It appears that you've been using this bridge with the neDB engine until now.
-        To continue using neDB, opt into it explicitly: add `matrix_appservice_slack_database_engine: nedb` to your vars.yml file and re-run this same command.
-        Alternatively, to migrate your existing neDB database to Postgres:
-        1. Stop all services (`ansible-playbook -i inventory/hosts setup.yml --tags=stop`)
-        2. Import the neDB database into Postgres (`ansible-playbook -v -i inventory/hosts setup.yml --tags=import-generic-sqlite-db --extra-vars='sqlite_database_path={{ matrix_appservice_slack_data_path }} postgres_connection_string_variable_name=matrix_appservice_slack_database_connString'`)
-        3. Re-run the playbook (`ansible-playbook -i inventory/hosts setup.yml --tags=setup-all,start`)
-    when: "matrix_appservice_slack_nedb_stat_result.stat.exists"
-  when: "matrix_appservice_slack_database_engine == 'postgres'"
diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
index 0ae137381b7..bf8072c1581 100644
--- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
+++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2
@@ -11,9 +11,10 @@ homeserver:
 
 {% if matrix_appservice_slack_database_engine == 'nedb' %}
 dbdir: "/data"
-{% endif %}
+{% else %}
 db:
-    engine: "{{ matrix_appservice_slack_database_engine }}"
-    connectionString: {{ matrix_appservice_slack_database_connString | to_json }}
+    engine: {{ matrix_appservice_slack_database_engine|to_json }}
+    connectionString: {{ matrix_appservice_slack_database_connectionString|to_json }}
+{% endif %}
 
 matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}"
diff --git a/roles/matrix-postgres/tasks/import_nedb.yml b/roles/matrix-postgres/tasks/import_nedb.yml
deleted file mode 100644
index 2a3dd587898..00000000000
--- a/roles/matrix-postgres/tasks/import_nedb.yml
+++ /dev/null
@@ -1,98 +0,0 @@
----
-
-# Pre-checks
-
-- name: Fail if Postgres not enabled
-  fail:
-    msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import."
-  when: "not matrix_postgres_enabled|bool"
-
-- name: Fail if playbook called incorrectly
-  fail:
-    msg: "The `nedb_database_path` variable needs to be provided to this playbook, via --extra-vars"
-  when: "nedb_database_path is not defined or nedb_database_path.startswith('<')"
-
-- name: Check if the provided nedb database file exists
-  stat:
-    path: "{{ nedb_database_path }}"
-  register: nedb_database_path_stat_result
-
-- name: Fail if provided SQLite database file doesn't exist
-  fail:
-    msg: "File cannot be found on the server at {{ nedb_database_path }}"
-  when: "not nedb_database_path_stat_result.stat.exists"
-
-# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string,
-# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string.
-
-- block:
-    - name: Fail if postgres_connection_string_variable_name points to an undefined variable
-      fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`"
-      when: "postgres_connection_string_variable_name not in vars"
-
-    - name: Get Postgres connection string from variable
-      set_fact:
-        postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}"
-  when: 'postgres_connection_string_variable_name is defined'
-
-- name: Fail if playbook called incorrectly
-  fail:
-    msg: >-
-      Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`.
-      Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:<port>/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"`
-  when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')"
-
-
-# Defaults
-
-- name: Set postgres_start_wait_time, if not provided
-  set_fact:
-    postgres_start_wait_time: 15
-  when: "postgres_start_wait_time|default('') == ''"
-
-
-# Actual import work
-
-- name: Ensure matrix-postgres is started
-  service:
-    name: matrix-postgres
-    state: started
-    daemon_reload: yes
-  register: matrix_postgres_service_start_result
-
-- name: Wait a bit, so that Postgres can start
-  wait_for:
-    timeout: "{{ postgres_start_wait_time }}"
-  delegate_to: 127.0.0.1
-  become: false
-  when: "matrix_postgres_service_start_result.changed|bool"
-
-# No migration.sh available, but found this:
-# https://github.com/matrix-org/matrix-appservice-slack/blob/develop/src/scripts/migrateToPostgres.ts
-# Usage should be similar to appservice_irc
-- name: Import appservice_slack NeDB database from {{ sqlite_database_path }} into Postgres
-  when: database == 'appservice_slack'
-  command:
-    cmd: >-
-      {{ matrix_host_command_docker }} run
-      --rm
-      --user={{ matrix_user_uid }}:{{ matrix_user_gid }}
-      --cap-drop=ALL
-      --network={{ matrix_docker_network }}
-      --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data,ro
-      --entrypoint=/bin/sh
-      {{ matrix_appservice_slack_docker_image }}
-      -c
-      'node /lib/scripts/migrate-db-to-pgres.js -d /data -p passkey.pem -c {{ postgres_db_connection_string }}'
-
-
-- name: Inject result
-  set_fact:
-    matrix_playbook_runtime_results: |
-      {{
-        matrix_playbook_runtime_results|default([])
-        +
-        [
-          "NOTE: Your NeDB database file has been imported into Postgres. The original directory has been moved from `{{ nedb_database_path }}` to `{{ nedb_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file."
-        ]
-      }}
diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml
index 86b8f8721e2..b9c2ae7c99c 100644
--- a/roles/matrix-postgres/tasks/main.yml
+++ b/roles/matrix-postgres/tasks/main.yml
@@ -32,14 +32,6 @@
   tags:
     - import-generic-sqlite-db
 
-# Imports slacks neDB to postgres.
-- import_tasks: "{{ role_path }}/tasks/import_nedb.yml"
-  vars:
-    database: appservice_slack
-  when: run_postgres_import_nedb|bool
-  tags:
-    - import-slack-nedb
-
 - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml"
   when: run_postgres_upgrade|bool
   tags:

From ad1425eee4f8d9eb1f92f5f0acecef8da3447280 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Wed, 23 Dec 2020 09:08:54 +0200
Subject: [PATCH 79/79] Add pgloader self-building support (for ARM)

---
 group_vars/matrix_servers                     |  2 +
 roles/matrix-postgres/defaults/main.yml       | 13 ++++-
 .../tasks/util/migrate_db_to_postgres.yml     | 55 +++++++++++++++----
 3 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers
index 4ddadb43229..c8af3cf2c0d 100755
--- a/group_vars/matrix_servers
+++ b/group_vars/matrix_servers
@@ -995,6 +995,8 @@ matrix_postgres_connection_username: "synapse"
 matrix_postgres_connection_password: "synapse-password"
 matrix_postgres_db_name: "homeserver"
 
+matrix_postgres_pgloader_container_image_self_build: "{{ matrix_architecture != 'amd64' }}"
+
 matrix_postgres_additional_databases: |
   {{
     ([{
diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml
index 0c5162812ac..8f1d0d78813 100644
--- a/roles/matrix-postgres/defaults/main.yml
+++ b/roles/matrix-postgres/defaults/main.yml
@@ -65,4 +65,15 @@ matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_post
 # For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all.
 matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15
 
-matrix_postgres_pgloader_docker_image: "docker.io/illagrenan/pgloader:3.6.2"
+
+matrix_postgres_pgloader_container_image_self_build: false
+matrix_postgres_pgloader_container_image_self_build_repo: "https://github.com/illagrenan/pgloader-docker.git"
+matrix_postgres_pgloader_container_image_self_build_repo_branch: "v{{ matrix_postgres_pgloader_docker_image_tag }}"
+matrix_postgres_pgloader_container_image_self_build_src_path: "{{ matrix_postgres_base_path }}/pgloader-container-src"
+
+# We use illagrenan/pgloader, instead of the more official dimitri/pgloader image,
+# because the official one only provides a `latest` tag.
+matrix_postgres_pgloader_docker_image: "{{ matrix_postgres_pgloader_docker_image_name_prefix }}illagrenan/pgloader:{{ matrix_postgres_pgloader_docker_image_tag }}"
+matrix_postgres_pgloader_docker_image_name_prefix: "{{ 'localhost/' if matrix_postgres_pgloader_container_image_self_build else 'docker.io/' }}"
+matrix_postgres_pgloader_docker_image_tag: "3.6.2"
+matrix_postgres_pgloader_docker_image_force_pull: "{{ matrix_postgres_pgloader_docker_image.endswith(':latest') }}"
diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
index d01611ef290..0da48c64eb4 100644
--- a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
+++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml
@@ -31,17 +31,50 @@
     msg: "File cannot be found on the server at {{ matrix_postgres_db_migration_request.src }}"
   when: "not matrix_postgres_db_migration_request_src_stat_result.stat.exists"
 
-- name: Fail if we cannot migrate on the current architecture ({{ matrix_architecture }})
-  fail:
-    msg: >-
-      {{ matrix_postgres_db_migration_request.engine_variable_name }} (part of {{ matrix_postgres_db_migration_request.caller }}) has been set to `postgres` (which is our new default now).
-      However, we've discovered an existing file-based database ({{ matrix_postgres_db_migration_request.engine_old }}) in {{ matrix_postgres_db_migration_request.src }}.
-      It appears that you've been using this bridge with a file-based database engine until now.
-      To continue using {{ matrix_postgres_db_migration_request.engine_old }}, opt into it explicitly: add `{{ matrix_postgres_db_migration_request.engine_variable_name }}: {{ matrix_postgres_db_migration_request.engine_old }}` to your vars.yml file and re-run this same command.
-      We'd normally auto-migrate you to Postgres, but we can't do it on the {{ matrix_architecture }} architecture. Our pgloader container image only supports amd64 (for now).
-      Learn more here: https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/740
-  when: "matrix_architecture != 'amd64'"
-
+- block:
+    - name: Ensure pgloader repository is present on self-build
+      git:
+        repo: "{{ matrix_postgres_pgloader_container_image_self_build_repo }}"
+        dest: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}"
+        version: "{{ matrix_postgres_pgloader_container_image_self_build_repo_branch }}"
+        force: "yes"
+      register: matrix_postgres_pgloader_git_pull_results
+
+    # If `stable` is used, we hit an error when processing /opt/src/pgloader/build/quicklisp/dists/quicklisp/software/uax-15-20201220-git/data/CompositionExclusions.txt:
+    # > the octet sequence #(194) cannot be decoded
+    #
+    # The issue is described here and is not getting fixed for months: https://github.com/dimitri/pgloader/pull/1179
+    #
+    # Although we're not using the dimitri/pgloader image, the one we're using suffers from the same problem.
+    - name: Switch pgloader base image from Debian stable (likely 10.x/Buster) to Bullseye
+      lineinfile:
+        path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}/Dockerfile"
+        regexp: "{{ item.match }}"
+        line: "{{ item.replace }}"
+      with_items:
+        - match: '^FROM debian:stable-slim as builder$'
+          replace: 'FROM debian:bullseye-slim as builder'
+        - match: '^FROM debian:stable-slim$'
+          replace: 'FROM debian:bullseye-slim'
+
+    - name: Ensure pgloader Docker image is built
+      docker_image:
+        name: "{{ matrix_postgres_pgloader_docker_image }}"
+        source: build
+        force_source: "{{ matrix_postgres_pgloader_git_pull_results.changed }}"
+        build:
+          dockerfile: Dockerfile
+          path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}"
+          pull: yes
+  when: "matrix_postgres_pgloader_container_image_self_build|bool"
+
+- name: Ensure pgloader Docker image is pulled
+  docker_image:
+    name: "{{ matrix_postgres_pgloader_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_postgres_pgloader_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_postgres_pgloader_docker_image_force_pull }}"
+  when: "not matrix_postgres_pgloader_container_image_self_build"
 
 # Defaults