diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index f5eb1cb8024..904b22919ce 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -47,12 +47,19 @@ matrix_appservice_discord_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_appservice_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'discord.as.token') | to_uuid }}" matrix_appservice_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'discord.hs.token') | to_uuid }}" +# We only make this use Postgres if our own Postgres server is enabled. +# It's only then (for now) that we can automatically create the necessary database and user for this service. +matrix_appservice_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_appservice_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.discord.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-appservice-discord @@ -121,6 +128,10 @@ matrix_appservice_slack_systemd_required_services_list: | (['matrix-synapse.service'] if matrix_synapse_enabled else []) }} +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_appservice_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_appservice_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.slack.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-appservice-slack @@ -156,6 +167,10 @@ matrix_appservice_irc_appservice_token: "{{ matrix_synapse_macaroon_secret_key | matrix_appservice_irc_homeserver_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'irc.hs.token') | to_uuid }}" +matrix_appservice_irc_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'nedb' }}" +matrix_appservice_irc_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.irc.db') | to_uuid }}" + + ###################################################################### # # /matrix-bridge-appservice-irc @@ -179,6 +194,8 @@ matrix_mautrix_facebook_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mautrix_facebook_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'fb.as.token') | to_uuid }}" @@ -189,6 +206,10 @@ matrix_mautrix_facebook_login_shared_secret: "{{ matrix_synapse_ext_password_pro matrix_mautrix_facebook_bridge_presence: "{{ matrix_synapse_use_presence if matrix_synapse_enabled else true }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mautrix_facebook_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mautrix_facebook_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_facebook.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mautrix-facebook @@ -212,6 +233,8 @@ matrix_mautrix_hangouts_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mautrix_hangouts_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ho.as.token') | to_uuid }}" @@ -222,6 +245,10 @@ matrix_mautrix_hangouts_container_http_host_bind_port: "{{ '' if matrix_nginx_pr matrix_mautrix_hangouts_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mautrix_hangouts_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mautrix_hangouts_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_hangouts.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mautrix-hangouts @@ -246,6 +273,8 @@ matrix_mautrix_telegram_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mautrix_telegram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'telegr.as.token') | to_uuid }}" @@ -258,6 +287,10 @@ matrix_mautrix_telegram_container_http_host_bind_port: "{{ '' if matrix_nginx_pr matrix_mautrix_telegram_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mautrix_telegram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mautrix_telegram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_telegram.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mautrix-telegram @@ -278,6 +311,8 @@ matrix_mautrix_whatsapp_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mautrix_whatsapp_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'whats.as.token') | to_uuid }}" @@ -286,6 +321,10 @@ matrix_mautrix_whatsapp_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mautrix_whatsapp_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mautrix_whatsapp_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mautrix_whatsapp_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mautrix_whatsapp.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mautrix-whatsapp @@ -334,6 +373,8 @@ matrix_mx_puppet_skype_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_skype_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'skype.as.tok') | to_uuid }}" @@ -342,6 +383,10 @@ matrix_mx_puppet_skype_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mx_puppet_skype_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_skype_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_skype_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_skype.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-skype @@ -365,6 +410,8 @@ matrix_mx_puppet_slack_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_slack_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxslk.as.tok') | to_uuid }}" @@ -373,6 +420,10 @@ matrix_mx_puppet_slack_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mx_puppet_slack_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_slack_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_slack_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_slack.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-slack @@ -395,6 +446,8 @@ matrix_mx_puppet_twitter_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_twitter_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxtwt.as.tok') | to_uuid }}" @@ -405,6 +458,10 @@ matrix_mx_puppet_twitter_login_shared_secret: "{{ matrix_synapse_ext_password_pr matrix_mx_puppet_twitter_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else ('127.0.0.1:' ~ matrix_mx_puppet_twitter_appservice_port) }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_twitter_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_twitter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_twitter.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-twitter @@ -428,6 +485,8 @@ matrix_mx_puppet_instagram_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_instagram_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxig.as.tok') | to_uuid }}" @@ -436,6 +495,10 @@ matrix_mx_puppet_instagram_homeserver_token: "{{ matrix_synapse_macaroon_secret_ matrix_mx_puppet_instagram_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_instagram_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_instagram_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_instagram.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-instagram @@ -458,6 +521,8 @@ matrix_mx_puppet_discord_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_discord_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxdsc.as.tok') | to_uuid }}" @@ -466,6 +531,10 @@ matrix_mx_puppet_discord_homeserver_token: "{{ matrix_synapse_macaroon_secret_ke matrix_mx_puppet_discord_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_discord_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_discord_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_discord.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-discord @@ -488,6 +557,8 @@ matrix_mx_puppet_steam_systemd_required_services_list: | ['docker.service'] + (['matrix-synapse.service'] if matrix_synapse_enabled else []) + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) }} matrix_mx_puppet_steam_appservice_token: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mxste.as.tok') | to_uuid }}" @@ -496,6 +567,10 @@ matrix_mx_puppet_steam_homeserver_token: "{{ matrix_synapse_macaroon_secret_key matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret if matrix_synapse_ext_password_provider_shared_secret_auth_enabled else '' }}" +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_mx_puppet_steam_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_mx_puppet_steam_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'as.mx_puppet_steam.db') | to_uuid }}" + ###################################################################### # # /matrix-bridge-mx-puppet-steam @@ -512,6 +587,17 @@ matrix_mx_puppet_steam_login_shared_secret: "{{ matrix_synapse_ext_password_prov # We don't enable bots by default. matrix_bot_matrix_reminder_bot_enabled: false +matrix_bot_matrix_reminder_bot_systemd_required_services_list: | + {{ + ['docker.service'] + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + }} + +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_bot_matrix_reminder_bot_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_bot_matrix_reminder_bot_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'reminder.bot.db') | to_uuid }}" + ###################################################################### # # /matrix-bot-matrix-reminder-bot @@ -606,6 +692,17 @@ matrix_dimension_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_ena matrix_integration_manager_rest_url: "{{ matrix_dimension_integrations_rest_url if matrix_dimension_enabled else None }}" matrix_integration_manager_ui_url: "{{ matrix_dimension_integrations_ui_url if matrix_dimension_enabled else None }}" +matrix_dimension_systemd_required_services_list: | + {{ + ['docker.service'] + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + }} + +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_dimension_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_dimension_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'dimension.db') | to_uuid }}" + ###################################################################### # # /matrix-dimension @@ -750,6 +847,11 @@ matrix_ma1sd_threepid_medium_email_connectors_smtp_tls: 0 matrix_ma1sd_self_check_validate_certificates: "{{ false if matrix_ssl_retrieval_method == 'self-signed' else true }}" +matrix_ma1sd_systemd_required_services_list: | + {{ + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + }} + matrix_ma1sd_systemd_wanted_services_list: | {{ (['matrix-corporal.service'] if matrix_corporal_enabled else ['matrix-synapse.service']) @@ -759,6 +861,10 @@ matrix_ma1sd_systemd_wanted_services_list: | (['matrix-mailer.service'] if matrix_mailer_enabled else []) }} +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_ma1sd_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_ma1sd_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'ma1sd.db') | to_uuid }}" + ###################################################################### # # /matrix-ma1sd @@ -875,6 +981,133 @@ matrix_postgres_connection_username: "synapse" matrix_postgres_connection_password: "synapse-password" matrix_postgres_db_name: "homeserver" +matrix_postgres_pgloader_container_image_self_build: "{{ matrix_architecture != 'amd64' }}" + +matrix_postgres_additional_databases: | + {{ + ([{ + 'name': matrix_ma1sd_database_name, + 'username': matrix_ma1sd_database_username, + 'password': matrix_ma1sd_database_password, + }] if (matrix_ma1sd_enabled and matrix_ma1sd_database_engine == 'postgres' and matrix_ma1sd_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_bot_matrix_reminder_bot_database_name, + 'username': matrix_bot_matrix_reminder_bot_database_username, + 'password': matrix_bot_matrix_reminder_bot_database_password, + }] if (matrix_bot_matrix_reminder_bot_enabled and matrix_bot_matrix_reminder_bot_database_engine == 'postgres' and matrix_bot_matrix_reminder_bot_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_registration_database_name, + 'username': matrix_registration_database_username, + 'password': matrix_registration_database_password, + }] if (matrix_registration_enabled and matrix_registration_database_engine == 'postgres' and matrix_registration_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_appservice_discord_database_name, + 'username': matrix_appservice_discord_database_username, + 'password': matrix_appservice_discord_database_password, + }] if (matrix_appservice_discord_enabled and matrix_appservice_discord_database_engine == 'postgres' and matrix_appservice_discord_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_appservice_slack_database_name, + 'username': matrix_appservice_slack_database_username, + 'password': matrix_appservice_slack_database_password, + }] if (matrix_appservice_slack_enabled and matrix_appservice_slack_database_engine == 'postgres' and matrix_appservice_slack_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_appservice_irc_database_name, + 'username': matrix_appservice_irc_database_username, + 'password': matrix_appservice_irc_database_password, + }] if (matrix_appservice_irc_enabled and matrix_appservice_irc_database_engine == 'postgres' and matrix_appservice_irc_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_mautrix_facebook_database_name, + 'username': matrix_mautrix_facebook_database_username, + 'password': matrix_mautrix_facebook_database_password, + }] if (matrix_mautrix_facebook_enabled and matrix_mautrix_facebook_database_engine == 'postgres' and matrix_mautrix_facebook_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_mautrix_hangouts_database_name, + 'username': matrix_mautrix_hangouts_database_username, + 'password': matrix_mautrix_hangouts_database_password, + }] if (matrix_mautrix_hangouts_enabled and matrix_mautrix_hangouts_database_engine == 'postgres' and matrix_mautrix_hangouts_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_mautrix_telegram_database_name, + 'username': matrix_mautrix_telegram_database_username, + 'password': matrix_mautrix_telegram_database_password, + }] if (matrix_mautrix_telegram_enabled and matrix_mautrix_telegram_database_engine == 'postgres' and matrix_mautrix_telegram_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_mautrix_whatsapp_database_name, + 'username': matrix_mautrix_whatsapp_database_username, + 'password': matrix_mautrix_whatsapp_database_password, + }] if (matrix_mautrix_whatsapp_enabled and matrix_mautrix_whatsapp_database_engine == 'postgres' and matrix_mautrix_whatsapp_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': 'matrix_bridge_sms', + 'username': 'matrix_bridge_sms', + 'password': matrix_synapse_macaroon_secret_key | password_hash('sha512', 'bridge_sms.db.secret') | to_uuid, + }] if matrix_sms_bridge_enabled else []) + + + ([{ + 'name': matrix_mx_puppet_skype_database_name, + 'username': matrix_mx_puppet_skype_database_username, + 'password': matrix_mx_puppet_skype_database_password, + }] if (matrix_mx_puppet_skype_enabled and matrix_mx_puppet_skype_database_engine == 'postgres' and matrix_mx_puppet_skype_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_mx_puppet_slack_database_name, + 'username': matrix_mx_puppet_slack_database_username, + 'password': matrix_mx_puppet_slack_database_password, + }] if (matrix_mx_puppet_slack_enabled and matrix_mx_puppet_slack_database_engine == 'postgres' and matrix_mx_puppet_slack_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_mx_puppet_twitter_database_name, + 'username': matrix_mx_puppet_twitter_database_username, + 'password': matrix_mx_puppet_twitter_database_password, + }] if (matrix_mx_puppet_twitter_enabled and matrix_mx_puppet_twitter_database_engine == 'postgres' and matrix_mx_puppet_twitter_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_mx_puppet_instagram_database_name, + 'username': matrix_mx_puppet_instagram_database_username, + 'password': matrix_mx_puppet_instagram_database_password, + }] if (matrix_mx_puppet_instagram_enabled and matrix_mx_puppet_instagram_database_engine == 'postgres' and matrix_mx_puppet_instagram_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_mx_puppet_discord_database_name, + 'username': matrix_mx_puppet_discord_database_username, + 'password': matrix_mx_puppet_discord_database_password, + }] if (matrix_mx_puppet_discord_enabled and matrix_mx_puppet_discord_database_engine == 'postgres' and matrix_mx_puppet_discord_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_mx_puppet_steam_database_name, + 'username': matrix_mx_puppet_steam_database_username, + 'password': matrix_mx_puppet_steam_database_password, + }] if (matrix_mx_puppet_steam_enabled and matrix_mx_puppet_steam_database_engine == 'postgres' and matrix_mx_puppet_steam_database_hostname == 'matrix-postgres') else []) + + + ([{ + 'name': matrix_dimension_database_name, + 'username': matrix_dimension_database_username, + 'password': matrix_dimension_database_password, + }] if (matrix_dimension_enabled and matrix_dimension_database_engine == 'postgres' and matrix_dimension_database_hostname == 'matrix-postgres') else []) + }} + +matrix_postgres_import_roles_to_ignore: | + {{ + [matrix_postgres_connection_username] + + + matrix_postgres_additional_databases|map(attribute='username') + }} + +matrix_postgres_import_databases_to_ignore: | + {{ + [matrix_postgres_db_name] + + + matrix_postgres_additional_databases|map(attribute='name') + }} + ###################################################################### # # /matrix-postgres @@ -1071,6 +1304,17 @@ matrix_registration_api_validate_certs: "{{ false if matrix_ssl_retrieval_method matrix_registration_container_image_self_build: "{{ matrix_architecture != 'amd64' }}" +matrix_registration_systemd_required_services_list: | + {{ + ['docker.service'] + + + (['matrix-postgres.service'] if matrix_postgres_enabled else []) + }} + +# Postgres is the default, except if not using `matrix_postgres` (internal postgres) +matrix_registration_database_engine: "{{ 'postgres' if matrix_postgres_enabled else 'sqlite' }}" +matrix_registration_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'mx.registr.db') | to_uuid }}" + ###################################################################### # # /matrix-registration diff --git a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml index 3302814732c..29bc83073a0 100644 --- a/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml +++ b/roles/matrix-bot-matrix-reminder-bot/defaults/main.yml @@ -21,6 +21,34 @@ matrix_bot_matrix_reminder_bot_systemd_required_services_list: ['docker.service' matrix_bot_matrix_reminder_bot_systemd_wanted_services_list: [] +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_bot_matrix_reminder_bot_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_bot_matrix_reminder_bot_database_*` variables +matrix_bot_matrix_reminder_bot_database_engine: 'sqlite' + +matrix_bot_matrix_reminder_bot_sqlite_database_path_local: "{{ matrix_bot_matrix_reminder_bot_data_path }}/bot.db" +matrix_bot_matrix_reminder_bot_sqlite_database_path_in_container: "/data/bot.db" + +matrix_bot_matrix_reminder_bot_database_username: 'matrix_reminder_bot' +matrix_bot_matrix_reminder_bot_database_password: 'some-password' +matrix_bot_matrix_reminder_bot_database_hostname: 'matrix-postgres' +matrix_bot_matrix_reminder_bot_database_port: 5432 +matrix_bot_matrix_reminder_bot_database_name: 'matrix_reminder_bot' + +matrix_bot_matrix_reminder_bot_database_connection_string: 'postgres://{{ matrix_bot_matrix_reminder_bot_database_username }}:{{ matrix_bot_matrix_reminder_bot_database_password }}@{{ matrix_bot_matrix_reminder_bot_database_hostname }}:{{ matrix_bot_matrix_reminder_bot_database_port }}/{{ matrix_bot_matrix_reminder_bot_database_name }}' + +matrix_bot_matrix_reminder_bot_storage_database: "{{ + { + 'sqlite': ('sqlite://' + matrix_bot_matrix_reminder_bot_sqlite_database_path_in_container), + 'postgres': matrix_bot_matrix_reminder_bot_database_connection_string, + }[matrix_bot_matrix_reminder_bot_database_engine] +}}" + + # The bot's username. This user needs to be created manually beforehand. # Also see `matrix_bot_matrix_reminder_bot_user_password`. matrix_bot_matrix_reminder_bot_matrix_user_id_localpart: "bot.matrix-reminder-bot" diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml index ade3d19119a..fc2afddb278 100644 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/main.yml @@ -8,7 +8,14 @@ - setup-all - setup-bot-matrix-reminder-bot -- import_tasks: "{{ role_path }}/tasks/setup.yml" +- import_tasks: "{{ role_path }}/tasks/setup_install.yml" + when: "run_setup|bool and matrix_bot_matrix_reminder_bot_enabled|bool" + tags: + - setup-all + - setup-bot-matrix-reminder-bot + +- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" + when: "run_setup|bool and not matrix_bot_matrix_reminder_bot_enabled|bool" tags: - setup-all - setup-bot-matrix-reminder-bot diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml deleted file mode 100644 index bc211e3103e..00000000000 --- a/roles/matrix-bot-matrix-reminder-bot/tasks/setup.yml +++ /dev/null @@ -1,88 +0,0 @@ ---- - -# -# Tasks related to setting up matrix-reminder-bot -# - -- name: Ensure matrix-reminder-bot paths exist - file: - path: "{{ item.path }}" - state: directory - mode: 0750 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - with_items: - - { path: "{{ matrix_bot_matrix_reminder_bot_config_path }}", when: true } - - { path: "{{ matrix_bot_matrix_reminder_bot_data_path }}", when: true } - - { path: "{{ matrix_bot_matrix_reminder_bot_data_store_path }}", when: true } - when: matrix_bot_matrix_reminder_bot_enabled|bool and item.when - -- name: Ensure matrix-reminder-bot image is pulled - docker_image: - name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_matrix_reminder_bot_docker_image_force_pull }}" - when: matrix_bot_matrix_reminder_bot_enabled|bool - -- name: Ensure matrix-reminder-bot config installed - copy: - content: "{{ matrix_bot_matrix_reminder_bot_configuration|to_nice_yaml }}" - dest: "{{ matrix_bot_matrix_reminder_bot_config_path }}/config.yaml" - mode: 0644 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_user_groupname }}" - when: matrix_bot_matrix_reminder_bot_enabled|bool - -- name: Ensure matrix-matrix-reminder-bot.service installed - template: - src: "{{ role_path }}/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2" - dest: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service" - mode: 0644 - register: matrix_bot_matrix_reminder_bot_systemd_service_result - when: matrix_bot_matrix_reminder_bot_enabled|bool - -- name: Ensure systemd reloaded after matrix-matrix-reminder-bot.service installation - service: - daemon_reload: yes - when: "matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_systemd_service_result.changed" - -# -# Tasks related to getting rid of matrix-reminder-bot (if it was previously enabled) -# - -- name: Check existence of matrix-matrix-reminder-bot service - stat: - path: "{{ matrix_systemd_path }}/matrix-matrix-reminder-bot.service" - register: matrix_bot_matrix_reminder_bot_service_stat - -- name: Ensure matrix-matrix-reminder-bot is stopped - service: - name: matrix-matrix-reminder-bot - state: stopped - daemon_reload: yes - register: stopping_result - when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists" - -- name: Ensure matrix-matrix-reminder-bot.service doesn't exist - file: - path: "{{ matrix_systemd_path }}/matrix-matrix-reminder-bot.service" - state: absent - when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists" - -- name: Ensure systemd reloaded after matrix-matrix-reminder-bot.service removal - service: - daemon_reload: yes - when: "not matrix_bot_matrix_reminder_bot_enabled|bool and matrix_bot_matrix_reminder_bot_service_stat.stat.exists" - -- name: Ensure Matrix matrix-reminder-bot paths don't exist - file: - path: "{{ matrix_bot_matrix_reminder_bot_base_path }}" - state: absent - when: "not matrix_bot_matrix_reminder_bot_enabled|bool" - -- name: Ensure matrix-reminder-bot Docker image doesn't exist - docker_image: - name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" - state: absent - when: "not matrix_bot_matrix_reminder_bot_enabled|bool" diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml new file mode 100644 index 00000000000..fea1e00ddff --- /dev/null +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_install.yml @@ -0,0 +1,73 @@ +--- + +- set_fact: + matrix_bot_matrix_reminder_bot_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}" + register: matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_bot_matrix_reminder_bot_sqlite_database_path_local }}" + dst: "{{ matrix_bot_matrix_reminder_bot_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_bot_matrix_reminder_bot_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-bot-matrix-reminder-bot.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_bot_matrix_reminder_bot_requires_restart: true + when: "matrix_bot_matrix_reminder_bot_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_bot_matrix_reminder_bot_database_engine == 'postgres'" + +- name: Ensure matrix-reminder-bot paths exist + file: + path: "{{ item.path }}" + state: directory + mode: 0750 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + with_items: + - { path: "{{ matrix_bot_matrix_reminder_bot_config_path }}", when: true } + - { path: "{{ matrix_bot_matrix_reminder_bot_data_path }}", when: true } + - { path: "{{ matrix_bot_matrix_reminder_bot_data_store_path }}", when: true } + when: "item.when|bool" + +- name: Ensure matrix-reminder-bot image is pulled + docker_image: + name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_bot_matrix_reminder_bot_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_bot_matrix_reminder_bot_docker_image_force_pull }}" + +- name: Ensure matrix-reminder-bot config installed + copy: + content: "{{ matrix_bot_matrix_reminder_bot_configuration|to_nice_yaml }}" + dest: "{{ matrix_bot_matrix_reminder_bot_config_path }}/config.yaml" + mode: 0644 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + +- name: Ensure matrix-bot-matrix-reminder-bot.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-bot-matrix-reminder-bot.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service" + mode: 0644 + register: matrix_bot_matrix_reminder_bot_systemd_service_result + +- name: Ensure systemd reloaded after matrix-bot-matrix-reminder-bot.service installation + service: + daemon_reload: yes + when: "matrix_bot_matrix_reminder_bot_systemd_service_result.changed|bool" + +- name: Ensure matrix-bot-matrix-reminder-bot.service restarted, if necessary + service: + name: "matrix-bot-matrix-reminder-bot.service" + state: restarted + when: "matrix_bot_matrix_reminder_bot_requires_restart|bool" diff --git a/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml new file mode 100644 index 00000000000..744f474d113 --- /dev/null +++ b/roles/matrix-bot-matrix-reminder-bot/tasks/setup_uninstall.yml @@ -0,0 +1,35 @@ +--- + +- name: Check existence of matrix-matrix-reminder-bot service + stat: + path: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service" + register: matrix_bot_matrix_reminder_bot_service_stat + +- name: Ensure matrix-matrix-reminder-bot is stopped + service: + name: matrix-matrix-reminder-bot + state: stopped + daemon_reload: yes + register: stopping_result + when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool" + +- name: Ensure matrix-bot-matrix-reminder-bot.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-bot-matrix-reminder-bot.service" + state: absent + when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool" + +- name: Ensure systemd reloaded after matrix-bot-matrix-reminder-bot.service removal + service: + daemon_reload: yes + when: "matrix_bot_matrix_reminder_bot_service_stat.stat.exists|bool" + +- name: Ensure Matrix matrix-reminder-bot paths don't exist + file: + path: "{{ matrix_bot_matrix_reminder_bot_base_path }}" + state: absent + +- name: Ensure matrix-reminder-bot Docker image doesn't exist + docker_image: + name: "{{ matrix_bot_matrix_reminder_bot_docker_image }}" + state: absent diff --git a/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2 b/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2 index e5ba5651727..59643958468 100644 --- a/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2 +++ b/roles/matrix-bot-matrix-reminder-bot/templates/config.yaml.j2 @@ -23,7 +23,7 @@ storage: # For Postgres, this would look like: # database: "postgres://username:password@localhost/dbname?sslmode=disable" #database: "postgres://matrix-reminder-bot:remindme@localhost/matrix-reminder-bot?sslmode=disable" - database: "sqlite:///data/bot.db" + database: {{ matrix_bot_matrix_reminder_bot_storage_database|to_json }} # The path to a directory for internal bot storage # containing encryption keys, sync tokens, etc. store_path: "/data/store" diff --git a/roles/matrix-bridge-appservice-discord/defaults/main.yml b/roles/matrix-bridge-appservice-discord/defaults/main.yml index 9ca06b052d4..c7cdddb6e50 100644 --- a/roles/matrix-bridge-appservice-discord/defaults/main.yml +++ b/roles/matrix-bridge-appservice-discord/defaults/main.yml @@ -41,6 +41,30 @@ matrix_appservice_discord_bridge_homeserverUrl: "{{ matrix_homeserver_url }}" matrix_appservice_discord_bridge_disablePresence: false matrix_appservice_discord_bridge_enableSelfServiceBridging: false +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_appservice_discord_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_appservice_discord_postgres_*` variables +matrix_appservice_discord_database_engine: 'sqlite' + +matrix_appservice_discord_sqlite_database_path_local: "{{ matrix_appservice_discord_data_path }}/discord.db" +matrix_appservice_discord_sqlite_database_path_in_container: "/data/discord.db" + +matrix_appservice_discord_database_username: 'matrix_appservice_discord' +matrix_appservice_discord_database_password: 'some-password' +matrix_appservice_discord_database_hostname: 'matrix-postgres' +matrix_appservice_discord_database_port: 5432 +matrix_appservice_discord_database_name: 'matrix_appservice_discord' + +# These 2 variables are what actually ends up in the bridge configuration. +# It's best if you don't change them directly, but rather redefine the sub-variables that constitute them. +matrix_appservice_discord_database_filename: "{{ matrix_appservice_discord_sqlite_database_path_in_container }}" +matrix_appservice_discord_database_connString: 'postgresql://{{ matrix_appservice_discord_database_username }}:{{ matrix_appservice_discord_database_password }}@{{ matrix_appservice_discord_database_hostname }}:{{ matrix_appservice_discord_database_port }}/{{ matrix_appservice_discord_database_name }}' + + # Tells whether the bot should make use of "Privileged Gateway Intents". # # Enabling this means that you need to enable it for the bot (Discord application) as well, diff --git a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml index 935d07ebfeb..8bf50e2804e 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/setup_install.yml @@ -1,5 +1,31 @@ --- +- set_fact: + matrix_appservice_discord_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_appservice_discord_sqlite_database_path_local }}" + register: matrix_appservice_discord_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_appservice_discord_sqlite_database_path_local }}" + dst: "{{ matrix_appservice_discord_database_connString }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_appservice_discord_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-appservice-discord.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_appservice_discord_requires_restart: true + when: "matrix_appservice_discord_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_appservice_discord_database_engine == 'postgres'" + - name: Ensure Appservice Discord image is pulled docker_image: name: "{{ matrix_appservice_discord_docker_image }}" @@ -80,3 +106,9 @@ service: daemon_reload: yes when: "matrix_appservice_discord_systemd_service_result.changed" + +- name: Ensure matrix-appservice-discord.service restarted, if necessary + service: + name: "matrix-appservice-discord.service" + state: restarted + when: "matrix_appservice_discord_requires_restart|bool" diff --git a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml index b0713a43ddb..73253ba030b 100644 --- a/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-discord/tasks/validate_config.yml @@ -20,3 +20,7 @@ when: "item.old in vars" with_items: - {'old': 'matrix_appservice_discord_container_expose_client_server_api_port', 'new': ''} + +- name: Require a valid database engine + fail: msg="`matrix_appservice_discord_database_engine` needs to be either 'sqlite' or 'postgres'" + when: "matrix_appservice_discord_database_engine not in ['sqlite', 'postgres']" diff --git a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 index b99f522abb9..b2ecd1981d3 100644 --- a/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-discord/templates/config.yaml.j2 @@ -58,8 +58,11 @@ database: # If you are migrating, see https://github.com/Half-Shot/matrix-appservice-discord/blob/master/docs/howto.md#migrate-to-postgres-from-sqlite # WARNING: You will almost certainly be fine with sqlite unless your bridge # is in heavy demand and you suffer from IO slowness. - filename: "/data/discord.db" - # connString: "postgresql://user:password@localhost/database_name" + {% if matrix_appservice_discord_database_engine == 'sqlite' %} + filename: {{ matrix_appservice_discord_database_filename|to_json }} + {% else %} + connString: {{ matrix_appservice_discord_database_connString|to_json }} + {% endif %} room: # Set the default visibility of alias rooms, defaults to "public". # One of: "public", "private" diff --git a/roles/matrix-bridge-appservice-irc/defaults/main.yml b/roles/matrix-bridge-appservice-irc/defaults/main.yml index 7ff2d46a37c..0b671e76299 100644 --- a/roles/matrix-bridge-appservice-irc/defaults/main.yml +++ b/roles/matrix-bridge-appservice-irc/defaults/main.yml @@ -16,6 +16,25 @@ matrix_appservice_irc_homeserver_domain: '{{ matrix_domain }}' matrix_appservice_irc_homeserver_enablePresence: true matrix_appservice_irc_appservice_address: 'http://matrix-appservice-irc:9999' +matrix_appservice_irc_database_engine: nedb +matrix_appservice_irc_database_username: matrix_appservice_irc +matrix_appservice_irc_database_password: ~ +matrix_appservice_irc_database_hostname: 'matrix-postgres' +matrix_appservice_irc_database_port: 5432 +matrix_appservice_irc_database_name: matrix_appservice_irc + +# This is just the Postgres connection string, if Postgres is used. +# Naming clashes with `matrix_appservice_irc_database_connectionString` somewhat. +matrix_appservice_irc_database_connection_string: 'postgresql://{{ matrix_appservice_irc_database_username }}:{{ matrix_appservice_irc_database_password }}@{{ matrix_appservice_irc_database_hostname }}:{{ matrix_appservice_irc_database_port }}/{{ matrix_appservice_irc_database_name }}?sslmode=disable' + +# This is what actually goes into `database.connectionString` for the bridge. +matrix_appservice_irc_database_connectionString: "{{ + { + 'nedb': 'nedb:///data', + 'postgres': matrix_appservice_irc_database_connection_string, + }[matrix_appservice_irc_database_engine] +}}" + matrix_appservice_irc_ircService_servers: [] # Example of `matrix_appservice_irc_ircService_servers` with one server (and all its options): diff --git a/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml new file mode 100644 index 00000000000..3fab195ab0a --- /dev/null +++ b/roles/matrix-bridge-appservice-irc/tasks/migrate_nedb_to_postgres.yml @@ -0,0 +1,64 @@ +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate." + when: "not matrix_postgres_enabled|bool" + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + +# Actual import work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +- name: Ensure matrix-appservice-irc is stopped + service: + name: matrix-appservice-irc + state: stopped + +- name: Import appservice-irc NeDB database into Postgres + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_appservice_irc_data_path }},dst=/data + --entrypoint=/bin/sh + {{ matrix_appservice_irc_docker_image }} + -c + '/usr/local/bin/node /app/lib/scripts/migrate-db-to-pgres.js --dbdir /data --privateKey /data/passkey.pem --connectionString {{ matrix_appservice_irc_database_connection_string }}' + +- name: Archive NeDB database files + command: + cmd: "mv {{ matrix_appservice_irc_data_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}.backup" + with_items: + - rooms.db + - users.db + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your appservice-irc database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_irc_data_path }}/*.db` to `{{ matrix_appservice_irc_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files." + ] + }} diff --git a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml index 5e313347fce..00568c0df7b 100644 --- a/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-irc/tasks/setup_install.yml @@ -1,12 +1,5 @@ --- -- name: Ensure Appservice IRC image is pulled - docker_image: - name: "{{ matrix_appservice_irc_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}" - - name: Ensure Appservice IRC paths exist file: path: "{{ item }}" @@ -24,25 +17,48 @@ path: "{{ matrix_appservice_irc_base_path }}/passkey.pem" register: matrix_appservice_irc_stat_passkey -- name: (Data relocation) Ensure matrix-appservice-irc.service is stopped - service: - name: matrix-appservice-irc - state: stopped - daemon_reload: yes - failed_when: false +- block: + - name: (Data relocation) Ensure matrix-appservice-irc.service is stopped + service: + name: matrix-appservice-irc + state: stopped + daemon_reload: yes + failed_when: false + + - name: (Data relocation) Move AppService IRC passkey.pem file to ./data directory + command: "mv {{ matrix_appservice_irc_base_path }}/passkey.pem {{ matrix_appservice_irc_data_path }}/passkey.pem" + + - name: (Data relocation) Move AppService IRC database files to ./data directory + command: "mv {{ matrix_appservice_irc_base_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}" + with_items: + - rooms.db + - users.db + failed_when: false when: "matrix_appservice_irc_stat_passkey.stat.exists" -- name: (Data relocation) Move AppService IRC passkey.pem file to ./data directory - command: "mv {{ matrix_appservice_irc_base_path }}/passkey.pem {{ matrix_appservice_irc_data_path }}/passkey.pem" - when: "matrix_appservice_irc_stat_passkey.stat.exists" +- set_fact: + matrix_appservice_irc_requires_restart: false -- name: (Data relocation) Move AppService IRC database files to ./data directory - command: "mv {{ matrix_appservice_irc_base_path }}/{{ item }} {{ matrix_appservice_irc_data_path }}/{{ item }}" - with_items: - - rooms.db - - users.db - failed_when: false - when: "matrix_appservice_irc_stat_passkey.stat.exists" +- block: + - name: Check if a nedb database already exists + stat: + path: "{{ matrix_appservice_irc_data_path }}/users.db" + register: matrix_appservice_irc_nedb_database_path_local_stat_result + + - block: + - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml" + + - set_fact: + matrix_appservice_irc_requires_restart: true + when: "matrix_appservice_irc_nedb_database_path_local_stat_result.stat.exists|bool" + when: "matrix_appservice_irc_database_engine == 'postgres'" + +- name: Ensure Appservice IRC image is pulled + docker_image: + name: "{{ matrix_appservice_irc_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_appservice_irc_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_irc_docker_image_force_pull }}" - name: Ensure Matrix Appservice IRC config installed copy: @@ -147,3 +163,9 @@ service: daemon_reload: yes when: "matrix_appservice_irc_systemd_service_result.changed" + +- name: Ensure matrix-appservice-irc.service restarted, if necessary + service: + name: "matrix-appservice-irc.service" + state: restarted + when: "matrix_appservice_irc_requires_restart|bool" diff --git a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 index 3daa1898764..94bbda7b812 100644 --- a/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-irc/templates/config.yaml.j2 @@ -127,8 +127,8 @@ advanced: # Use an external database to store bridge state. database: # database engine (must be 'postgres' or 'nedb'). Default: nedb - engine: "nedb" + engine: {{ matrix_appservice_irc_database_engine|to_json }} # Either a PostgreSQL connection string, or a path to the NeDB storage directory. # For postgres, it must start with postgres:// # For NeDB, it must start with nedb://. The path is relative to the project directory. - connectionString: "nedb:///data" + connectionString: {{ matrix_appservice_irc_database_connectionString|to_json }} diff --git a/roles/matrix-bridge-appservice-slack/defaults/main.yml b/roles/matrix-bridge-appservice-slack/defaults/main.yml index f2cfd8f9361..28646a3c4a8 100644 --- a/roles/matrix-bridge-appservice-slack/defaults/main.yml +++ b/roles/matrix-bridge-appservice-slack/defaults/main.yml @@ -45,6 +45,26 @@ matrix_appservice_slack_appservice_token: '' matrix_appservice_slack_homeserver_token: '' matrix_appservice_slack_id_token: '' +matrix_appservice_slack_database_engine: nedb +matrix_appservice_slack_database_username: matrix_appservice_slack +matrix_appservice_slack_database_password: ~ +matrix_appservice_slack_database_hostname: 'matrix-postgres' +matrix_appservice_slack_database_port: 5432 +matrix_appservice_slack_database_name: matrix_appservice_slack + +# This is just the Postgres connection string, if Postgres is used. +# Naming clashes with `matrix_appservice_slack_database_connectionString` somewhat. +matrix_appservice_slack_database_connection_string: 'postgresql://{{ matrix_appservice_slack_database_username }}:{{ matrix_appservice_slack_database_password }}@{{ matrix_appservice_slack_database_hostname }}:{{ matrix_appservice_slack_database_port }}/{{ matrix_appservice_slack_database_name }}?sslmode=disable' + +# This is what actually goes into `database.connectionString` for the bridge. +matrix_appservice_slack_database_connectionString: "{{ + { + 'nedb': 'nedb:///data', + 'postgres': matrix_appservice_slack_database_connection_string, + }[matrix_appservice_slack_database_engine] +}}" + + matrix_appservice_slack_configuration_yaml: "{{ lookup('template', 'templates/config.yaml.j2') }}" matrix_appservice_slack_configuration_extension_yaml: | diff --git a/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml b/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml new file mode 100644 index 00000000000..fedad977571 --- /dev/null +++ b/roles/matrix-bridge-appservice-slack/tasks/migrate_nedb_to_postgres.yml @@ -0,0 +1,66 @@ +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate." + when: "not matrix_postgres_enabled|bool" + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + +# Actual import work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +- name: Ensure matrix-appservice-slack is stopped + service: + name: matrix-appservice-slack + state: stopped + +- name: Import appservice-slack NeDB database into Postgres + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_appservice_slack_data_path }},dst=/data + --entrypoint=/bin/sh + {{ matrix_appservice_slack_docker_image }} + -c + '/usr/local/bin/node /usr/src/app/lib/scripts/migrateToPostgres.js --dbdir /data --connectionString {{ matrix_appservice_slack_database_connection_string }}' + +- name: Archive NeDB database files + command: + cmd: "mv {{ matrix_appservice_slack_data_path }}/{{ item }} {{ matrix_appservice_slack_data_path }}/{{ item }}.backup" + with_items: + - teams.db + - room-store.db + - user-store.db + - event-store.db + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your appservice-slack database files have been imported into Postgres. The original database files have been moved from `{{ matrix_appservice_slack_data_path }}/*.db` to `{{ matrix_appservice_slack_data_path }}/*.db.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete these files." + ] + }} diff --git a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml index 94e0fedf92a..29b0f39e9bf 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/setup_install.yml @@ -1,12 +1,5 @@ --- -- name: Ensure Appservice Slack image is pulled - docker_image: - name: "{{ matrix_appservice_slack_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}" - - name: Ensure AppService Slack paths exist file: path: "{{ item }}" @@ -19,6 +12,30 @@ - "{{ matrix_appservice_slack_config_path }}" - "{{ matrix_appservice_slack_data_path }}" +- set_fact: + matrix_appservice_slack_requires_restart: false + +- block: + - name: Check if a nedb database already exists + stat: + path: "{{ matrix_appservice_slack_data_path }}/teams.db" + register: matrix_appservice_slack_nedb_database_path_local_stat_result + + - block: + - import_tasks: "{{ role_path }}/tasks/migrate_nedb_to_postgres.yml" + + - set_fact: + matrix_appservice_slack_requires_restart: true + when: "matrix_appservice_slack_nedb_database_path_local_stat_result.stat.exists|bool" + when: "matrix_appservice_slack_database_engine == 'postgres'" + +- name: Ensure Appservice Slack image is pulled + docker_image: + name: "{{ matrix_appservice_slack_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_appservice_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_appservice_slack_docker_image_force_pull }}" + - name: Ensure Matrix Appservice Slack config installed copy: content: "{{ matrix_appservice_slack_configuration|to_nice_yaml }}" @@ -46,3 +63,9 @@ service: daemon_reload: yes when: "matrix_appservice_slack_systemd_service_result.changed" + +- name: Ensure matrix-appservice-slack.service restarted, if necessary + service: + name: "matrix-appservice-slack.service" + state: restarted + when: "matrix_appservice_slack_requires_restart|bool" diff --git a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml index 5da5f947e1e..8af10f2f54c 100644 --- a/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml +++ b/roles/matrix-bridge-appservice-slack/tasks/validate_config.yml @@ -9,4 +9,4 @@ - "matrix_appservice_slack_control_room_id" - "matrix_appservice_slack_appservice_token" - "matrix_appservice_slack_homeserver_token" - - "matrix_appservice_slack_id_token" \ No newline at end of file + - "matrix_appservice_slack_id_token" diff --git a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 index 8f48d3178ca..bf8072c1581 100644 --- a/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-appservice-slack/templates/config.yaml.j2 @@ -9,6 +9,12 @@ homeserver: url: "{{ matrix_appservice_slack_homeserver_url }}" server_name: "{{ matrix_domain }}" +{% if matrix_appservice_slack_database_engine == 'nedb' %} dbdir: "/data" +{% else %} +db: + engine: {{ matrix_appservice_slack_database_engine|to_json }} + connectionString: {{ matrix_appservice_slack_database_connectionString|to_json }} +{% endif %} matrix_admin_room: "{{ matrix_appservice_slack_control_room_id }}" diff --git a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml index 1547ad71f56..580934dba42 100644 --- a/roles/matrix-bridge-mautrix-facebook/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-facebook/defaults/main.yml @@ -32,6 +32,35 @@ matrix_mautrix_facebook_systemd_wanted_services_list: [] matrix_mautrix_facebook_appservice_token: '' matrix_mautrix_facebook_homeserver_token: '' + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_mautrix_facebook_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_mautrix_facebook_postgres_*` variables +matrix_mautrix_facebook_database_engine: 'sqlite' + +matrix_mautrix_facebook_sqlite_database_path_local: "{{ matrix_mautrix_facebook_data_path }}/mautrix-facebook.db" +matrix_mautrix_facebook_sqlite_database_path_in_container: "/data/mautrix-facebook.db" + +matrix_mautrix_facebook_database_username: 'matrix_mautrix_facebook' +matrix_mautrix_facebook_database_password: 'some-password' +matrix_mautrix_facebook_database_hostname: 'matrix-postgres' +matrix_mautrix_facebook_database_port: 5432 +matrix_mautrix_facebook_database_name: 'matrix_mautrix_facebook' + +matrix_mautrix_facebook_database_connection_string: 'postgresql://{{ matrix_mautrix_facebook_database_username }}:{{ matrix_mautrix_facebook_database_password }}@{{ matrix_mautrix_facebook_database_hostname }}:{{ matrix_mautrix_facebook_database_port }}/{{ matrix_mautrix_facebook_database_name }}' + +matrix_mautrix_facebook_appservice_database: "{{ + { + 'sqlite': ('sqlite://' + matrix_mautrix_facebook_sqlite_database_path_in_container), + 'postgres': matrix_mautrix_facebook_database_connection_string, + }[matrix_mautrix_facebook_database_engine] +}}" + + # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mautrix_facebook_login_shared_secret: '' diff --git a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml index 638854e2c5b..7f3104467fb 100644 --- a/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-facebook/tasks/setup_install.yml @@ -8,6 +8,32 @@ The matrix-bridge-mautrix-facebook role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" +- set_fact: + matrix_mautrix_facebook_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}" + register: matrix_mautrix_facebook_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mautrix_facebook_sqlite_database_path_local }}" + dst: "{{ matrix_mautrix_facebook_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mautrix_facebook_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mautrix-facebook.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mautrix_facebook_requires_restart: true + when: "matrix_mautrix_facebook_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mautrix_facebook_database_engine == 'postgres'" + - name: Ensure Mautrix Facebook image is pulled docker_image: name: "{{ matrix_mautrix_facebook_docker_image }}" @@ -94,3 +120,9 @@ service: daemon_reload: yes when: "matrix_mautrix_facebook_systemd_service_result.changed" + +- name: Ensure matrix-mautrix-facebook.service restarted, if necessary + service: + name: "matrix-mautrix-facebook.service" + state: restarted + when: "matrix_mautrix_facebook_requires_restart|bool" diff --git a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 index 9d8de2d5ba5..09287362ff9 100644 --- a/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-facebook/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: sqlite:////data/mautrix-facebook.db + database: {{ matrix_mautrix_facebook_appservice_database|to_json }} # Public part of web server for out-of-Matrix interaction with the bridge. public: diff --git a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 index 1818465099a..caa52eb570b 100644 --- a/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 +++ b/roles/matrix-bridge-mautrix-facebook/templates/systemd/matrix-mautrix-facebook.service.j2 @@ -18,6 +18,7 @@ ExecStartPre={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-fac --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ -v {{ matrix_mautrix_facebook_config_path }}:/config:z \ -v {{ matrix_mautrix_facebook_data_path }}:/data:z \ {{ matrix_mautrix_facebook_docker_image }} \ diff --git a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml index ab670052853..984bec4862d 100644 --- a/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-hangouts/defaults/main.yml @@ -39,6 +39,35 @@ matrix_mautrix_hangouts_systemd_wanted_services_list: [] matrix_mautrix_hangouts_appservice_token: '' matrix_mautrix_hangouts_homeserver_token: '' + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_mautrix_hangouts_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_mautrix_hangouts_postgres_*` variables +matrix_mautrix_hangouts_database_engine: 'sqlite' + +matrix_mautrix_hangouts_sqlite_database_path_local: "{{ matrix_mautrix_hangouts_data_path }}/mautrix-hangouts.db" +matrix_mautrix_hangouts_sqlite_database_path_in_container: "/data/mautrix-hangouts.db" + +matrix_mautrix_hangouts_database_username: 'matrix_mautrix_hangouts' +matrix_mautrix_hangouts_database_password: 'some-password' +matrix_mautrix_hangouts_database_hostname: 'matrix-postgres' +matrix_mautrix_hangouts_database_port: 5432 +matrix_mautrix_hangouts_database_name: 'matrix_mautrix_hangouts' + +matrix_mautrix_hangouts_database_connection_string: 'postgresql://{{ matrix_mautrix_hangouts_database_username }}:{{ matrix_mautrix_hangouts_database_password }}@{{ matrix_mautrix_hangouts_database_hostname }}:{{ matrix_mautrix_hangouts_database_port }}/{{ matrix_mautrix_hangouts_database_name }}' + +matrix_mautrix_hangouts_appservice_database: "{{ + { + 'sqlite': ('sqlite://' + matrix_mautrix_hangouts_sqlite_database_path_in_container), + 'postgres': matrix_mautrix_hangouts_database_connection_string, + }[matrix_mautrix_hangouts_database_engine] +}}" + + # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mautrix_hangouts_login_shared_secret: '' diff --git a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml index eec5e00675a..15af96265a4 100644 --- a/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-hangouts/tasks/setup_install.yml @@ -8,6 +8,32 @@ The matrix-bridge-mautrix-hangouts role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" +- set_fact: + matrix_mautrix_hangouts_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}" + register: matrix_mautrix_hangouts_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mautrix_hangouts_sqlite_database_path_local }}" + dst: "{{ matrix_mautrix_hangouts_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mautrix_hangouts_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mautrix-hangouts.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mautrix_hangouts_requires_restart: true + when: "matrix_mautrix_hangouts_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mautrix_hangouts_database_engine == 'postgres'" + - name: Ensure Mautrix Hangouts image is pulled docker_image: name: "{{ matrix_mautrix_hangouts_docker_image }}" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mautrix_hangouts_systemd_service_result.changed" + +- name: Ensure matrix-mautrix-hangouts.service restarted, if necessary + service: + name: "matrix-mautrix-hangouts.service" + state: restarted + when: "matrix_mautrix_hangouts_requires_restart|bool" diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 index f274b2034c6..cc2ca90bb99 100644 --- a/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-hangouts/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: sqlite:////data/mautrix-hangouts.db + database: {{ matrix_mautrix_hangouts_appservice_database|to_json }} # The unique ID of this appservice. id: hangouts diff --git a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 index 26280da32af..395591905f5 100644 --- a/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 +++ b/roles/matrix-bridge-mautrix-hangouts/templates/systemd/matrix-mautrix-hangouts.service.j2 @@ -18,6 +18,7 @@ ExecStartPre={{ matrix_host_command_docker }} run --rm --name matrix-mautrix-han --log-driver=none \ --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ -v {{ matrix_mautrix_hangouts_config_path }}:/config:z \ -v {{ matrix_mautrix_hangouts_data_path }}:/data:z \ {{ matrix_mautrix_hangouts_docker_image }} \ diff --git a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml index b6f4ef5bb35..ceebd3ec13c 100644 --- a/roles/matrix-bridge-mautrix-telegram/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-telegram/defaults/main.yml @@ -43,6 +43,35 @@ matrix_mautrix_telegram_systemd_wanted_services_list: [] matrix_mautrix_telegram_appservice_token: '' matrix_mautrix_telegram_homeserver_token: '' + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_mautrix_telegram_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_mautrix_telegram_postgres_*` variables +matrix_mautrix_telegram_database_engine: 'sqlite' + +matrix_mautrix_telegram_sqlite_database_path_local: "{{ matrix_mautrix_telegram_data_path }}/mautrix-telegram.db" +matrix_mautrix_telegram_sqlite_database_path_in_container: "/data/mautrix-telegram.db" + +matrix_mautrix_telegram_database_username: 'matrix_mautrix_telegram' +matrix_mautrix_telegram_database_password: 'some-password' +matrix_mautrix_telegram_database_hostname: 'matrix-postgres' +matrix_mautrix_telegram_database_port: 5432 +matrix_mautrix_telegram_database_name: 'matrix_mautrix_telegram' + +matrix_mautrix_telegram_database_connection_string: 'postgresql://{{ matrix_mautrix_telegram_database_username }}:{{ matrix_mautrix_telegram_database_password }}@{{ matrix_mautrix_telegram_database_hostname }}:{{ matrix_mautrix_telegram_database_port }}/{{ matrix_mautrix_telegram_database_name }}' + +matrix_mautrix_telegram_appservice_database: "{{ + { + 'sqlite': ('sqlite://' + matrix_mautrix_telegram_sqlite_database_path_in_container), + 'postgres': matrix_mautrix_telegram_database_connection_string, + }[matrix_mautrix_telegram_database_engine] +}}" + + # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mautrix_telegram_login_shared_secret: '' diff --git a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml index 9dc38ec2c3f..6a37974a833 100644 --- a/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-telegram/tasks/setup_install.yml @@ -8,6 +8,32 @@ The matrix-bridge-mautrix-telegram role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" +- set_fact: + matrix_mautrix_telegram_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}" + register: matrix_mautrix_telegram_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mautrix_telegram_sqlite_database_path_local }}" + dst: "{{ matrix_mautrix_telegram_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mautrix_telegram_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mautrix-telegram.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mautrix_telegram_requires_restart: true + when: "matrix_mautrix_telegram_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mautrix_telegram_database_engine == 'postgres'" + - name: Ensure Mautrix Telegram image is pulled docker_image: name: "{{ matrix_mautrix_telegram_docker_image }}" @@ -71,3 +97,9 @@ service: daemon_reload: yes when: "matrix_mautrix_telegram_systemd_service_result.changed" + +- name: Ensure matrix-mautrix-telegram.service restarted, if necessary + service: + name: "matrix-mautrix-telegram.service" + state: restarted + when: "matrix_mautrix_telegram_requires_restart|bool" diff --git a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 index 1a4ac43e9b5..52efba02d73 100644 --- a/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-telegram/templates/config.yaml.j2 @@ -27,7 +27,7 @@ appservice: # Format examples: # SQLite: sqlite:///filename.db # Postgres: postgres://username:password@hostname/dbname - database: sqlite:////data/mautrix-telegram.db + database: {{ matrix_mautrix_telegram_appservice_database|to_json }} # Public part of web server for out-of-Matrix interaction with the bridge. # Used for things like login if the user wants to make sure the 2FA password isn't stored in diff --git a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml index e9929f96ead..0d4f9852774 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/defaults/main.yml @@ -27,6 +27,42 @@ matrix_mautrix_whatsapp_systemd_wanted_services_list: [] matrix_mautrix_whatsapp_appservice_token: '' matrix_mautrix_whatsapp_homeserver_token: '' + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_mautrix_whatsapp_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_mautrix_whatsapp_postgres_*` variables +matrix_mautrix_whatsapp_database_engine: 'sqlite' + +matrix_mautrix_whatsapp_sqlite_database_path_local: "{{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db" +matrix_mautrix_whatsapp_sqlite_database_path_in_container: "/data/mautrix-whatsapp.db" + +matrix_mautrix_whatsapp_database_username: 'matrix_mautrix_whatsapp' +matrix_mautrix_whatsapp_database_password: 'some-password' +matrix_mautrix_whatsapp_database_hostname: 'matrix-postgres' +matrix_mautrix_whatsapp_database_port: 5432 +matrix_mautrix_whatsapp_database_name: 'matrix_mautrix_whatsapp' + +matrix_mautrix_whatsapp_database_connection_string: 'postgresql://{{ matrix_mautrix_whatsapp_database_username }}:{{ matrix_mautrix_whatsapp_database_password }}@{{ matrix_mautrix_whatsapp_database_hostname }}:{{ matrix_mautrix_whatsapp_database_port }}/{{ matrix_mautrix_whatsapp_database_name }}' + +matrix_mautrix_whatsapp_appservice_database_type: "{{ + { + 'sqlite': 'sqlite3', + 'postgres':'postgres', + }[matrix_mautrix_whatsapp_database_engine] +}}" + +matrix_mautrix_whatsapp_appservice_database_uri: "{{ + { + 'sqlite': matrix_mautrix_whatsapp_sqlite_database_path_in_container, + 'postgres': matrix_mautrix_whatsapp_database_connection_string, + }[matrix_mautrix_whatsapp_database_engine] +}}" + + # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mautrix_whatsapp_login_shared_secret: '' diff --git a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml index 6bd87bbe2c1..8d894a8493a 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml +++ b/roles/matrix-bridge-mautrix-whatsapp/tasks/setup_install.yml @@ -8,6 +8,32 @@ The matrix-bridge-mautrix-whatsapp role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" +- set_fact: + matrix_mautrix_whatsapp_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}" + register: matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mautrix_whatsapp_sqlite_database_path_local }}" + dst: "{{ matrix_mautrix_whatsapp_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mautrix_whatsapp_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mautrix-whatsapp.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mautrix_whatsapp_requires_restart: true + when: "matrix_mautrix_whatsapp_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mautrix_whatsapp_database_engine == 'postgres'" + - name: Ensure Mautrix Whatsapp image is pulled docker_image: name: "{{ matrix_mautrix_whatsapp_docker_image }}" @@ -26,12 +52,12 @@ - "{{ matrix_mautrix_whatsapp_base_path }}" - "{{ matrix_mautrix_whatsapp_config_path }}" - "{{ matrix_mautrix_whatsapp_data_path }}" - + - name: Check if an old database file exists stat: path: "{{ matrix_mautrix_whatsapp_base_path }}/mautrix-whatsapp.db" register: matrix_mautrix_whatsapp_stat_database - + - name: Check if an old matrix state file exists stat: path: "{{ matrix_mautrix_whatsapp_base_path }}/mx-state.json" @@ -48,7 +74,7 @@ - name: (Data relocation) Move mautrix-whatsapp database file to ./data directory command: "mv {{ matrix_mautrix_whatsapp_base_path }}/mautrix-whatsapp.db {{ matrix_mautrix_whatsapp_data_path }}/mautrix-whatsapp.db" when: "matrix_mautrix_whatsapp_stat_database.stat.exists" - + - name: (Data relocation) Move mautrix-whatsapp mx-state file to ./data directory command: "mv {{ matrix_mautrix_whatsapp_base_path }}/mx-state.json {{ matrix_mautrix_whatsapp_data_path }}/mx-state.json" when: "matrix_mautrix_whatsapp_stat_mx_state.stat.exists" @@ -80,3 +106,9 @@ service: daemon_reload: yes when: "matrix_mautrix_whatsapp_systemd_service_result.changed" + +- name: Ensure matrix-mautrix-whatsapp.service restarted, if necessary + service: + name: "matrix-mautrix-whatsapp.service" + state: restarted + when: "matrix_mautrix_whatsapp_requires_restart|bool" diff --git a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 index a527a1884fc..8921669567a 100644 --- a/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mautrix-whatsapp/templates/config.yaml.j2 @@ -19,11 +19,11 @@ appservice: # Database config. database: # The database type. "sqlite3" and "postgres" are supported. - type: sqlite3 + type: {{ matrix_mautrix_whatsapp_appservice_database_type|to_json }} # The database URI. # SQLite: File name is enough. https://github.com/mattn/go-sqlite3#connection-string # Postgres: Connection string. For example, postgres://user:password@host/database - uri: mautrix-whatsapp.db + uri: {{ matrix_mautrix_whatsapp_appservice_database_uri|to_json }} # Maximum number of connections. Mostly relevant for Postgres. max_open_conns: 20 max_idle_conns: 2 diff --git a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml index 268bbf8f8c0..97b203137ee 100644 --- a/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-discord/defaults/main.yml @@ -55,6 +55,20 @@ matrix_mx_puppet_discord_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_discord_login_shared_secret: '' +# Database configuration +matrix_mx_puppet_discord_database_engine: 'sqlite' + +matrix_mx_puppet_discord_sqlite_database_path_local: "{{ matrix_mx_puppet_discord_data_path }}/database.db" +matrix_mx_puppet_discord_sqlite_database_path_in_container: "/data/database.db" + +matrix_mx_puppet_discord_database_username: matrix_mx_puppet_discord +matrix_mx_puppet_discord_database_password: ~ +matrix_mx_puppet_discord_database_hostname: 'matrix-postgres' +matrix_mx_puppet_discord_database_port: 5432 +matrix_mx_puppet_discord_database_name: matrix_mx_puppet_discord + +matrix_mx_puppet_discord_database_connection_string: 'postgresql://{{ matrix_mx_puppet_discord_database_username }}:{{ matrix_mx_puppet_discord_database_password }}@{{ matrix_mx_puppet_discord_database_hostname }}:{{ matrix_mx_puppet_discord_database_port }}/{{ matrix_mx_puppet_discord_database_name }}?sslmode=disable' + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml index c3f2b01fbd7..3e3b2f9447e 100644 --- a/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-discord/tasks/setup_install.yml @@ -8,14 +8,6 @@ The matrix-bridge-mx-puppet-discord role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- name: Ensure MX Puppet Discord image is pulled - docker_image: - name: "{{ matrix_mx_puppet_discord_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_mx_puppet_discord_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_discord_docker_image_force_pull }}" - when: matrix_mx_puppet_discord_enabled|bool and not matrix_mx_puppet_discord_container_image_self_build - - name: Ensure MX Puppet Discord paths exist file: path: "{{ item.path }}" @@ -30,6 +22,57 @@ - { path: "{{ matrix_mx_puppet_discord_docker_src_files_path }}", when: "{{ matrix_mx_puppet_discord_container_image_self_build }}" } when: matrix_mx_puppet_discord_enabled|bool and item.when|bool +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_discord_base_path }}/database.db" + register: matrix_mx_puppet_discord_stat_database + +- block: + - name: (Data relocation) Ensure matrix-mx-puppet-discord.service is stopped + service: + name: matrix-mx-puppet-discord + state: stopped + daemon_reload: yes + failed_when: False + + - name: (Data relocation) Move mx-puppet-discord database file to ./data directory + command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db" + when: "matrix_mx_puppet_discord_stat_database.stat.exists" + +- set_fact: + matrix_mx_puppet_discord_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_discord_sqlite_database_path_local }}" + register: matrix_mx_puppet_discord_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_discord_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_discord_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_discord_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-discord.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_discord_requires_restart: true + when: "matrix_mx_puppet_discord_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_discord_database_engine == 'postgres'" + +- name: Ensure MX Puppet Discord image is pulled + docker_image: + name: "{{ matrix_mx_puppet_discord_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mx_puppet_discord_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_discord_docker_image_force_pull }}" + when: matrix_mx_puppet_discord_enabled|bool and not matrix_mx_puppet_discord_container_image_self_build + - name: Ensure MX Puppet Discord repository is present on self build git: repo: "{{ matrix_mx_puppet_discord_container_image_self_build_repo }}" @@ -49,23 +92,6 @@ pull: yes when: "matrix_mx_puppet_discord_enabled|bool and matrix_mx_puppet_discord_container_image_self_build|bool" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_discord_base_path }}/database.db" - register: matrix_mx_puppet_discord_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-discord.service is stopped - service: - name: matrix-mx-puppet-discord - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_discord_stat_database.stat.exists" - -- name: (Data relocation) Move mx-puppet-discord database file to ./data directory - command: "mv {{ matrix_mx_puppet_discord_base_path }}/database.db {{ matrix_mx_puppet_discord_data_path }}/database.db" - when: "matrix_mx_puppet_discord_stat_database.stat.exists" - - name: Ensure mx-puppet-discord config.yaml installed copy: content: "{{ matrix_mx_puppet_discord_configuration|to_nice_yaml }}" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_discord_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-discord.service restarted, if necessary + service: + name: "matrix-mx-puppet-discord.service" + state: restarted + when: "matrix_mx_puppet_discord_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 index 2c70379617d..1f4548d8353 100644 --- a/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-discord/templates/config.yaml.j2 @@ -100,15 +100,18 @@ namePatterns: group: :name database: +{% if matrix_mx_puppet_discord_database_engine == 'sqlite' %} + # Use SQLite3 as a database backend + # The name of the database file + filename: {{ matrix_mx_puppet_discord_sqlite_database_path_in_container|to_json }} +{% else %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" - # Use SQLite3 as a database backend - # The name of the database file - filename: /data/database.db + connString: {{ matrix_mx_puppet_discord_database_connection_string|to_json }} +{% endif %} logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml index 21cde53fa5b..cd08c010a27 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/defaults/main.yml @@ -46,6 +46,19 @@ matrix_mx_puppet_instagram_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_instagram_login_shared_secret: '' +matrix_mx_puppet_instagram_database_engine: sqlite + +matrix_mx_puppet_instagram_sqlite_database_path_local: "{{ matrix_mx_puppet_instagram_data_path }}/database.db" +matrix_mx_puppet_instagram_sqlite_database_path_in_container: "/data/database.db" + +matrix_mx_puppet_instagram_database_username: matrix_mx_puppet_instagram +matrix_mx_puppet_instagram_database_password: ~ +matrix_mx_puppet_instagram_database_hostname: 'matrix-postgres' +matrix_mx_puppet_instagram_database_port: 5432 +matrix_mx_puppet_instagram_database_name: matrix_mx_puppet_instagram + +matrix_mx_puppet_instagram_database_connection_string: 'postgresql://{{ matrix_mx_puppet_instagram_database_username }}:{{ matrix_mx_puppet_instagram_database_password }}@{{ matrix_mx_puppet_instagram_database_hostname }}:{{ matrix_mx_puppet_instagram_database_port }}/{{ matrix_mx_puppet_instagram_database_name }}?sslmode=disable' + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml index 78bd78c1697..76bbd629b80 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-instagram/tasks/setup_install.yml @@ -8,6 +8,33 @@ The matrix-bridge-mx-puppet-instagram role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" + +- set_fact: + matrix_mx_puppet_instagram_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_instagram_sqlite_database_path_local }}" + register: matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_instagram_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_instagram_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_instagram_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-instagram.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_instagram_requires_restart: true + when: "matrix_mx_puppet_instagram_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_instagram_database_engine == 'postgres'" + - name: Ensure mx-puppet-instagram image is pulled docker_image: name: "{{ matrix_mx_puppet_instagram_docker_image }}" @@ -77,4 +104,8 @@ daemon_reload: yes when: "matrix_mx_puppet_instagram_systemd_service_result.changed" - +- name: Ensure matrix-mx-puppet-instagram.service restarted, if necessary + service: + name: "matrix-mx-puppet-instagram.service" + state: restarted + when: "matrix_mx_puppet_instagram_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 index 634fbaecdc9..b830da2b682 100644 --- a/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-instagram/templates/config.yaml.j2 @@ -44,15 +44,18 @@ provisioning: apiPrefix: /_matrix/provision database: +{% if matrix_mx_puppet_instagram_database_engine == 'postgres' %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: {{ matrix_mx_puppet_instagram_database_connection_string|to_json }} +{% else %} # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + filename: {{ matrix_mx_puppet_instagram_sqlite_database_path_in_container|to_json }} +{% endif %} logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml index 0c16a8babea..83cd3dc55b4 100644 --- a/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-skype/defaults/main.yml @@ -53,6 +53,20 @@ matrix_mx_puppet_skype_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_skype_login_shared_secret: '' +# Database configuration, role default is `sqlite` but playbook default is `postgres` +matrix_mx_puppet_skype_database_engine: sqlite + +matrix_mx_puppet_skype_sqlite_database_path_local: "{{ matrix_mx_puppet_skype_data_path }}/database.db" +matrix_mx_puppet_skype_sqlite_database_path_in_container: "/data/database.db" + +matrix_mx_puppet_skype_database_username: matrix_mx_puppet_skype +matrix_mx_puppet_skype_database_password: ~ +matrix_mx_puppet_skype_database_hostname: 'matrix-postgres' +matrix_mx_puppet_skype_database_port: 5432 +matrix_mx_puppet_skype_database_name: matrix_mx_puppet_skype + +matrix_mx_puppet_skype_database_connection_string: 'postgresql://{{ matrix_mx_puppet_skype_database_username }}:{{ matrix_mx_puppet_skype_database_password }}@{{ matrix_mx_puppet_skype_database_hostname }}:{{ matrix_mx_puppet_skype_database_port }}/{{ matrix_mx_puppet_skype_database_name }}?sslmode=disable' + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml index fb5185eda22..9289a793413 100644 --- a/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-skype/tasks/setup_install.yml @@ -8,14 +8,6 @@ The matrix-bridge-mx-puppet-skype role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- name: Ensure MX Puppet Skype image is pulled - docker_image: - name: "{{ matrix_mx_puppet_skype_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_mx_puppet_skype_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_skype_docker_image_force_pull }}" - when: matrix_mx_puppet_skype_enabled|bool and not matrix_mx_puppet_skype_container_image_self_build - - name: Ensure MX Puppet Skype paths exist file: path: "{{ item.path }}" @@ -30,6 +22,57 @@ - { path: "{{ matrix_mx_puppet_skype_docker_src_files_path }}", when: "{{ matrix_mx_puppet_skype_container_image_self_build }}" } when: matrix_mx_puppet_skype_enabled|bool and item.when|bool +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_skype_base_path }}/database.db" + register: matrix_mx_puppet_skype_stat_database + +- name: (Data relocation) Ensure matrix-mx-puppet-skype.service is stopped + service: + name: matrix-mx-puppet-skype + state: stopped + daemon_reload: yes + failed_when: false + when: "matrix_mx_puppet_skype_stat_database.stat.exists" + +- name: (Data relocation) Move mx-puppet-skype database file to ./data directory + command: "mv {{ matrix_mx_puppet_skype_base_path }}/database.db {{ matrix_mx_puppet_skype_data_path }}/database.db" + when: "matrix_mx_puppet_skype_stat_database.stat.exists" + +- set_fact: + matrix_mx_puppet_skype_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_skype_sqlite_database_path_local }}" + register: matrix_mx_puppet_skype_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_skype_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_skype_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_skype_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-skype.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_skype_requires_restart: true + when: "matrix_mx_puppet_skype_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_skype_database_engine == 'postgres'" + +- name: Ensure MX Puppet Skype image is pulled + docker_image: + name: "{{ matrix_mx_puppet_skype_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mx_puppet_skype_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_skype_docker_image_force_pull }}" + when: matrix_mx_puppet_skype_enabled|bool and not matrix_mx_puppet_skype_container_image_self_build + - name: Ensure MX Puppet Skype repository is present on self build git: repo: "{{ matrix_mx_puppet_skype_container_image_self_build_repo }}" @@ -49,23 +92,6 @@ pull: yes when: "matrix_mx_puppet_skype_enabled|bool and matrix_mx_puppet_skype_container_image_self_build|bool" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_skype_base_path }}/database.db" - register: matrix_mx_puppet_skype_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-skype.service is stopped - service: - name: matrix-mx-puppet-skype - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_skype_stat_database.stat.exists" - -- name: (Data relocation) Move mx-puppet-skype database file to ./data directory - command: "mv {{ matrix_mx_puppet_skype_base_path }}/database.db {{ matrix_mx_puppet_skype_data_path }}/database.db" - when: "matrix_mx_puppet_skype_stat_database.stat.exists" - - name: Ensure mx-puppet-skype config.yaml installed copy: content: "{{ matrix_mx_puppet_skype_configuration|to_nice_yaml }}" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_skype_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-skype.service restarted, if necessary + service: + name: "matrix-mx-puppet-skype.service" + state: restarted + when: "matrix_mx_puppet_skype_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 index c7b5c870e62..d41d3a23568 100644 --- a/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-skype/templates/config.yaml.j2 @@ -68,15 +68,18 @@ logging: # - Store database: +{% if matrix_mx_puppet_skype_database_engine == 'postgres' %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: {{ matrix_mx_puppet_skype_database_connection_string|to_json }} +{% else %} # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + filename: {{ matrix_mx_puppet_skype_sqlite_database_path_in_container|to_json }} +{% endif %} provisioning: # Regex of Matrix IDs allowed to use the puppet bridge diff --git a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml index 96ab7833160..70b98eceebb 100644 --- a/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-slack/defaults/main.yml @@ -57,6 +57,20 @@ matrix_mx_puppet_slack_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_slack_login_shared_secret: '' +# Database configuration, role uses 'sqlite' per default but playbook sets up postgres by default +matrix_mx_puppet_slack_database_engine: sqlite + +matrix_mx_puppet_slack_sqlite_database_path_local: "{{ matrix_mx_puppet_slack_data_path }}/database.db" +matrix_mx_puppet_slack_sqlite_database_path_in_container: "/data/database.db" + +matrix_mx_puppet_slack_database_username: matrix_mx_puppet_slack +matrix_mx_puppet_slack_database_password: ~ +matrix_mx_puppet_slack_database_hostname: 'matrix-postgres' +matrix_mx_puppet_slack_database_port: 5432 +matrix_mx_puppet_slack_database_name: matrix_mx_puppet_slack + +matrix_mx_puppet_slack_database_connection_string: 'postgresql://{{ matrix_mx_puppet_slack_database_username }}:{{ matrix_mx_puppet_slack_database_password }}@{{ matrix_mx_puppet_slack_database_hostname }}:{{ matrix_mx_puppet_slack_database_port }}/{{ matrix_mx_puppet_slack_database_name }}?sslmode=disable' + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml index 2e45ecf6070..5d68435cdad 100644 --- a/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-slack/tasks/setup_install.yml @@ -8,14 +8,6 @@ The matrix-bridge-mx-puppet-slack role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- name: Ensure MX Puppet Slack image is pulled - docker_image: - name: "{{ matrix_mx_puppet_slack_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_mx_puppet_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_slack_docker_image_force_pull }}" - when: matrix_mx_puppet_slack_enabled|bool and not matrix_mx_puppet_slack_container_image_self_build - - name: Ensure MX Puppet Slack paths exist file: path: "{{ item.path }}" @@ -30,6 +22,53 @@ - { path: "{{ matrix_mx_puppet_slack_docker_src_files_path }}", when: "{{ matrix_mx_puppet_slack_container_image_self_build }}" } when: matrix_mx_puppet_slack_enabled|bool and item.when|bool +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_slack_base_path }}/database.db" + register: matrix_mx_puppet_slack_stat_database + +- name: (Data relocation) Ensure matrix-mx-puppet-slack.service is stopped + service: + name: matrix-mx-puppet-slack + state: stopped + daemon_reload: yes + failed_when: false + when: "matrix_mx_puppet_slack_stat_database.stat.exists" + +- set_fact: + matrix_mx_puppet_slack_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_slack_sqlite_database_path_local }}" + register: matrix_mx_puppet_slack_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_slack_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_slack_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_slack_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-slack.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_slack_requires_restart: true + when: "matrix_mx_puppet_slack_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_slack_database_engine == 'postgres'" + +- name: Ensure MX Puppet Slack image is pulled + docker_image: + name: "{{ matrix_mx_puppet_slack_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mx_puppet_slack_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_slack_docker_image_force_pull }}" + when: matrix_mx_puppet_slack_enabled|bool and not matrix_mx_puppet_slack_container_image_self_build + - name: Ensure MX Puppet Slack repository is present on self build git: repo: "{{ matrix_mx_puppet_slack_container_image_self_build_repo }}" @@ -49,19 +88,6 @@ pull: yes when: "matrix_mx_puppet_slack_enabled|bool and matrix_mx_puppet_slack_container_image_self_build" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_slack_base_path }}/database.db" - register: matrix_mx_puppet_slack_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-slack.service is stopped - service: - name: matrix-mx-puppet-slack - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_slack_stat_database.stat.exists" - - name: (Data relocation) Move mx-puppet-slack database file to ./data directory command: "mv {{ matrix_mx_puppet_slack_base_path }}/database.db {{ matrix_mx_puppet_slack_data_path }}/database.db" when: "matrix_mx_puppet_slack_stat_database.stat.exists" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_slack_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-slack.service restarted, if necessary + service: + name: "matrix-mx-puppet-slack.service" + state: restarted + when: "matrix_mx_puppet_slack_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 index b6e88784831..af6b5cb88db 100644 --- a/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-slack/templates/config.yaml.j2 @@ -58,15 +58,18 @@ provisioning: apiPrefix: /_matrix/provision database: +{% if matrix_mx_puppet_slack_database_engine == 'postgres' %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: {{ matrix_mx_puppet_slack_database_connection_string|to_json }} +{% else %} # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + filename: {{ matrix_mx_puppet_slack_sqlite_database_path_in_container|to_json }} +{% endif %} logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml index b5eb8473115..15fa889f0e9 100644 --- a/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-steam/defaults/main.yml @@ -55,6 +55,19 @@ matrix_mx_puppet_steam_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_steam_login_shared_secret: '' +matrix_mx_puppet_steam_database_engine: sqlite + +matrix_mx_puppet_steam_sqlite_database_path_local: "{{ matrix_mx_puppet_steam_data_path }}/database.db" +matrix_mx_puppet_steam_sqlite_database_path_in_container: "/data/database.db" + +matrix_mx_puppet_steam_database_username: matrix_mx_puppet_steam +matrix_mx_puppet_steam_database_password: ~ +matrix_mx_puppet_steam_database_hostname: 'matrix-postgres' +matrix_mx_puppet_steam_database_port: 5432 +matrix_mx_puppet_steam_database_name: matrix_mx_puppet_steam + +matrix_mx_puppet_steam_database_connection_string: 'postgresql://{{ matrix_mx_puppet_steam_database_username }}:{{ matrix_mx_puppet_steam_database_password }}@{{ matrix_mx_puppet_steam_database_hostname }}:{{ matrix_mx_puppet_steam_database_port }}/{{ matrix_mx_puppet_steam_database_name }}?sslmode=disable' + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml index 7b7f8211581..71f6d8897f9 100644 --- a/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-steam/tasks/setup_install.yml @@ -8,14 +8,6 @@ The matrix-bridge-mx-puppet-steam role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- name: Ensure MX Puppet Steam image is pulled - docker_image: - name: "{{ matrix_mx_puppet_steam_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_steam_docker_image_force_pull }}" - when: matrix_mx_puppet_steam_enabled|bool and not matrix_mx_puppet_steam_container_image_self_build - - name: Ensure MX Puppet Steam paths exist file: path: "{{ item.path }}" @@ -30,6 +22,57 @@ - { path: "{{ matrix_mx_puppet_steam_docker_src_files_path }}", when: "{{ matrix_mx_puppet_steam_container_image_self_build }}" } when: matrix_mx_puppet_steam_enabled|bool and item.when|bool +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_steam_base_path }}/database.db" + register: matrix_mx_puppet_steam_stat_database + +- name: (Data relocation) Ensure matrix-mx-puppet-steam.service is stopped + service: + name: matrix-mx-puppet-steam + state: stopped + daemon_reload: yes + failed_when: false + when: "matrix_mx_puppet_steam_stat_database.stat.exists" + +- name: (Data relocation) Move mx-puppet-steam database file to ./data directory + command: "mv {{ matrix_mx_puppet_steam_base_path }}/database.db {{ matrix_mx_puppet_steam_data_path }}/database.db" + when: "matrix_mx_puppet_steam_stat_database.stat.exists" + +- set_fact: + matrix_mx_puppet_steam_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_steam_sqlite_database_path_local }}" + register: matrix_mx_puppet_steam_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_steam_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_steam_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_steam_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-steam.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_steam_requires_restart: true + when: "matrix_mx_puppet_steam_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_steam_database_engine == 'postgres'" + +- name: Ensure MX Puppet Steam image is pulled + docker_image: + name: "{{ matrix_mx_puppet_steam_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mx_puppet_steam_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_steam_docker_image_force_pull }}" + when: matrix_mx_puppet_steam_enabled|bool and not matrix_mx_puppet_steam_container_image_self_build + - name: Ensure MX Puppet Steam repository is present on self build git: repo: "{{ matrix_mx_puppet_steam_container_image_self_build_repo }}" @@ -49,23 +92,6 @@ pull: yes when: "matrix_mx_puppet_steam_enabled|bool and matrix_mx_puppet_steam_container_image_self_build" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_steam_base_path }}/database.db" - register: matrix_mx_puppet_steam_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-steam.service is stopped - service: - name: matrix-mx-puppet-steam - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_steam_stat_database.stat.exists" - -- name: (Data relocation) Move mx-puppet-steam database file to ./data directory - command: "mv {{ matrix_mx_puppet_steam_base_path }}/database.db {{ matrix_mx_puppet_steam_data_path }}/database.db" - when: "matrix_mx_puppet_steam_stat_database.stat.exists" - - name: Ensure mx-puppet-steam config.yaml installed copy: content: "{{ matrix_mx_puppet_steam_configuration|to_nice_yaml }}" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_steam_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-steam.service restarted, if necessary + service: + name: "matrix-mx-puppet-steam.service" + state: restarted + when: "matrix_mx_puppet_steam_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 index d08982ca9f7..149e08b69a3 100644 --- a/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-steam/templates/config.yaml.j2 @@ -61,15 +61,18 @@ selfService: blacklist: {{ matrix_mx_puppet_steam_provisioning_blacklist|to_json }} database: +{% if matrix_mx_puppet_steam_database_engine == 'postgres' %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: {{ matrix_mx_puppet_steam_database_connection_string|to_json }} +{% else %} # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + filename: {{ matrix_mx_puppet_steam_sqlite_database_path_in_container|to_json }} +{% endif %} logging: # Log level of console output diff --git a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml index 13438a21f28..28639fdadb4 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/defaults/main.yml @@ -60,6 +60,20 @@ matrix_mx_puppet_twitter_homeserver_token: '' # Can be set to enable automatic double-puppeting via Shared Secret Auth (https://github.com/devture/matrix-synapse-shared-secret-auth). matrix_mx_puppet_twitter_login_shared_secret: '' +# Database configuration +matrix_mx_puppet_twitter_database_engine: sqlite + +matrix_mx_puppet_twitter_sqlite_database_path_local: "{{ matrix_mx_puppet_twitter_data_path }}/database.db" +matrix_mx_puppet_twitter_sqlite_database_path_in_container: "/data/database.db" + +matrix_mx_puppet_twitter_database_username: mx_puppet_twitter +matrix_mx_puppet_twitter_database_password: ~ +matrix_mx_puppet_twitter_database_hostname: 'matrix-postgres' +matrix_mx_puppet_twitter_database_port: 5432 +matrix_mx_puppet_twitter_database_name: matrix_mx_puppet_twitter + +matrix_mx_puppet_twitter_database_connection_string: 'postgresql://{{ matrix_mx_puppet_twitter_database_username }}:{{ matrix_mx_puppet_twitter_database_password }}@{{ matrix_mx_puppet_twitter_database_hostname }}:{{ matrix_mx_puppet_twitter_database_port }}/{{ matrix_mx_puppet_twitter_database_name }}?sslmode=disable' + # Default configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml index 663f822c498..3893981a565 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml +++ b/roles/matrix-bridge-mx-puppet-twitter/tasks/setup_install.yml @@ -8,14 +8,6 @@ The matrix-bridge-mx-puppet-twitter role needs to execute before the matrix-synapse role. when: "matrix_synapse_role_executed|default(False)" -- name: Ensure MX Puppet Twitter image is pulled - docker_image: - name: "{{ matrix_mx_puppet_twitter_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_mx_puppet_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_twitter_docker_image_force_pull }}" - when: matrix_mx_puppet_twitter_enabled|bool and not matrix_mx_puppet_twitter_container_image_self_build - - name: Ensure MX Puppet Twitter paths exist file: path: "{{ item.path }}" @@ -30,6 +22,57 @@ - { path: "{{ matrix_mx_puppet_twitter_docker_src_files_path }}", when: "{{ matrix_mx_puppet_twitter_container_image_self_build }}" } when: matrix_mx_puppet_twitter_enabled|bool and item.when|bool +- name: Check if an old database file already exists + stat: + path: "{{ matrix_mx_puppet_twitter_base_path }}/database.db" + register: matrix_mx_puppet_twitter_stat_database + +- name: (Data relocation) Ensure matrix-mx-puppet-twitter.service is stopped + service: + name: matrix-mx-puppet-twitter + state: stopped + daemon_reload: yes + failed_when: false + when: "matrix_mx_puppet_twitter_stat_database.stat.exists" + +- name: (Data relocation) Move mx-puppet-twitter database file to ./data directory + command: "mv {{ matrix_mx_puppet_twitter_base_path }}/database.db {{ matrix_mx_puppet_twitter_data_path }}/database.db" + when: "matrix_mx_puppet_twitter_stat_database.stat.exists" + +- set_fact: + matrix_mx_puppet_twitter_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_mx_puppet_twitter_sqlite_database_path_local }}" + register: matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_mx_puppet_twitter_sqlite_database_path_local }}" + dst: "{{ matrix_mx_puppet_twitter_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_mx_puppet_twitter_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-mx-puppet-twitter.service'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_mx_puppet_twitter_requires_restart: true + when: "matrix_mx_puppet_twitter_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_mx_puppet_twitter_database_engine == 'postgres'" + +- name: Ensure MX Puppet Twitter image is pulled + docker_image: + name: "{{ matrix_mx_puppet_twitter_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_mx_puppet_twitter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_mx_puppet_twitter_docker_image_force_pull }}" + when: matrix_mx_puppet_twitter_enabled|bool and not matrix_mx_puppet_twitter_container_image_self_build + - name: Ensure MX Puppet Twitter repository is present on self build git: repo: "{{ matrix_mx_puppet_twitter_container_image_self_build_repo }}" @@ -49,23 +92,6 @@ pull: yes when: "matrix_mx_puppet_twitter_enabled|bool and matrix_mx_puppet_twitter_container_image_self_build" -- name: Check if an old database file already exists - stat: - path: "{{ matrix_mx_puppet_twitter_base_path }}/database.db" - register: matrix_mx_puppet_twitter_stat_database - -- name: (Data relocation) Ensure matrix-mx-puppet-twitter.service is stopped - service: - name: matrix-mx-puppet-twitter - state: stopped - daemon_reload: yes - failed_when: false - when: "matrix_mx_puppet_twitter_stat_database.stat.exists" - -- name: (Data relocation) Move mx-puppet-twitter database file to ./data directory - command: "mv {{ matrix_mx_puppet_twitter_base_path }}/database.db {{ matrix_mx_puppet_twitter_data_path }}/database.db" - when: "matrix_mx_puppet_twitter_stat_database.stat.exists" - - name: Ensure mx-puppet-twitter config.yaml installed copy: content: "{{ matrix_mx_puppet_twitter_configuration|to_nice_yaml }}" @@ -93,3 +119,9 @@ service: daemon_reload: yes when: "matrix_mx_puppet_twitter_systemd_service_result.changed" + +- name: Ensure matrix-mx-puppet-twitter.service restarted, if necessary + service: + name: "matrix-mx-puppet-twitter.service" + state: restarted + when: "matrix_mx_puppet_twitter_requires_restart|bool" diff --git a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 index 7d3033b39d9..bdecf1dce07 100644 --- a/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 +++ b/roles/matrix-bridge-mx-puppet-twitter/templates/config.yaml.j2 @@ -54,15 +54,18 @@ provisioning: apiPrefix: /_matrix/provision database: +{% if matrix_mx_puppet_twitter_database_engine == 'postgres' %} # Use Postgres as a database backend # If set, will be used instead of SQLite3 # Connection string to connect to the Postgres instance # with username "user", password "pass", host "localhost" and database name "dbname". # Modify each value as necessary - #connString: "postgres://user:pass@localhost/dbname?sslmode=disable" + connString: {{ matrix_mx_puppet_twitter_database_connection_string|to_json }} +{% else %} # Use SQLite3 as a database backend # The name of the database file - filename: /data/database.db + filename: {{ matrix_mx_puppet_twitter_sqlite_database_path_in_container|to_json }} +{% endif %} logging: # Log level of console output diff --git a/roles/matrix-dimension/defaults/main.yml b/roles/matrix-dimension/defaults/main.yml index 3b69227abd1..8a691946f73 100644 --- a/roles/matrix-dimension/defaults/main.yml +++ b/roles/matrix-dimension/defaults/main.yml @@ -15,6 +15,12 @@ matrix_dimension_base_path: "{{ matrix_base_data_path }}/dimension" matrix_dimension_docker_image: "docker.io/turt2live/matrix-dimension:latest" matrix_dimension_docker_image_force_pull: "{{ matrix_dimension_docker_image.endswith(':latest') }}" +# List of systemd services that matrix-dimension.service depends on. +matrix_dimension_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-dimension.service wants +matrix_dimension_systemd_wanted_services_list: [] + # The user and group id correspond to the node user in the `turt2live/matrix-dimension` image. matrix_dimension_user_uid: '1000' matrix_dimension_user_gid: '1000' @@ -34,6 +40,28 @@ matrix_dimension_integrations_jitsi_widget_url: "https://{{ matrix_server_fqn_di matrix_dimension_homeserver_federationUrl: "http://matrix-synapse:8048" + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_dimension_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_dimension_postgres_*` variables +matrix_dimension_database_engine: 'sqlite' + +matrix_dimension_sqlite_database_path_local: "{{ matrix_dimension_base_path }}/dimension.db" +matrix_dimension_sqlite_database_path_in_container: "dimension.db" + +matrix_dimension_database_username: 'matrix_dimension' +matrix_dimension_database_password: 'some-password' +matrix_dimension_database_hostname: 'matrix-postgres' +matrix_dimension_database_port: 5432 +matrix_dimension_database_name: 'matrix_dimension' + +matrix_dimension_database_connection_string: 'postgres://{{ matrix_dimension_database_username }}:{{ matrix_dimension_database_password }}@{{ matrix_dimension_database_hostname }}:{{ matrix_dimension_database_port }}/{{ matrix_dimension_database_name }}' + + # Default Dimension configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-dimension/tasks/main.yml b/roles/matrix-dimension/tasks/main.yml index 1888f9452f1..aad552866a2 100644 --- a/roles/matrix-dimension/tasks/main.yml +++ b/roles/matrix-dimension/tasks/main.yml @@ -8,8 +8,14 @@ - setup-all - setup-dimension -- import_tasks: "{{ role_path }}/tasks/setup_dimension.yml" - when: run_setup|bool +- import_tasks: "{{ role_path }}/tasks/setup_install.yml" + when: run_setup|bool and matrix_dimension_enabled|bool + tags: + - setup-all + - setup-dimension + +- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" + when: run_setup|bool and not matrix_dimension_enabled|bool tags: - setup-all - setup-dimension diff --git a/roles/matrix-dimension/tasks/setup_dimension.yml b/roles/matrix-dimension/tasks/setup_dimension.yml deleted file mode 100644 index 2437a547223..00000000000 --- a/roles/matrix-dimension/tasks/setup_dimension.yml +++ /dev/null @@ -1,85 +0,0 @@ ---- - -# -# Tasks related to setting up the dimension -# - -- name: Ensure Dimension base path exists - file: - path: "{{ matrix_dimension_base_path }}" - state: directory - mode: 0770 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_dimension_user_gid }}" - when: matrix_dimension_enabled|bool - -- name: Ensure Dimension config installed - copy: - content: "{{ matrix_dimension_configuration|to_nice_yaml }}" - dest: "{{ matrix_dimension_base_path }}/config.yaml" - mode: 0640 - owner: "{{ matrix_user_username }}" - group: "{{ matrix_dimension_user_gid }}" - when: matrix_dimension_enabled|bool - -- name: Ensure Dimension image is pulled - docker_image: - name: "{{ matrix_dimension_docker_image }}" - source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" - force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" - force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_dimension_docker_image_force_pull }}" - when: matrix_dimension_enabled|bool - -- name: Ensure matrix-dimension.service installed - template: - src: "{{ role_path }}/templates/systemd/matrix-dimension.service.j2" - dest: "{{ matrix_systemd_path }}/matrix-dimension.service" - mode: 0644 - register: matrix_dimension_systemd_service_result - when: matrix_dimension_enabled|bool - -- name: Ensure systemd reloaded after matrix-dimension.service installation - service: - daemon_reload: yes - when: "matrix_dimension_enabled|bool and matrix_dimension_systemd_service_result.changed" - -# -# Tasks related to getting rid of the dimension (if it was previously enabled) -# - -- name: Check existence of matrix-dimension service - stat: - path: "{{ matrix_systemd_path }}/matrix-dimension.service" - register: matrix_dimension_service_stat - when: "not matrix_dimension_enabled|bool" - -- name: Ensure matrix-dimension is stopped - service: - name: matrix-dimension - state: stopped - daemon_reload: yes - register: stopping_result - when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists" - -- name: Ensure matrix-dimension.service doesn't exist - file: - path: "{{ matrix_systemd_path }}/matrix-dimension.service" - state: absent - when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists" - -- name: Ensure systemd reloaded after matrix-dimension.service removal - service: - daemon_reload: yes - when: "not matrix_dimension_enabled|bool and matrix_dimension_service_stat.stat.exists" - -- name: Ensure Dimension environment variables path doesn't exist - file: - path: "{{ matrix_dimension_base_path }}" - state: absent - when: "not matrix_dimension_enabled|bool" - -- name: Ensure Dimension Docker image doesn't exist - docker_image: - name: "{{ matrix_dimension_docker_image }}" - state: absent - when: "not matrix_dimension_enabled|bool" diff --git a/roles/matrix-dimension/tasks/setup_install.yml b/roles/matrix-dimension/tasks/setup_install.yml new file mode 100644 index 00000000000..9a264449674 --- /dev/null +++ b/roles/matrix-dimension/tasks/setup_install.yml @@ -0,0 +1,69 @@ +--- + +- set_fact: + matrix_dimension_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_dimension_sqlite_database_path_local }}" + register: matrix_dimension_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_dimension_sqlite_database_path_local }}" + dst: "{{ matrix_dimension_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_dimension_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-dimension.service'] + pgloader_options: ['--with "quote identifiers"'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_dimension_requires_restart: true + when: "matrix_dimension_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_dimension_database_engine == 'postgres'" + +- name: Ensure Dimension base path exists + file: + path: "{{ matrix_dimension_base_path }}" + state: directory + mode: 0770 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_dimension_user_gid }}" + +- name: Ensure Dimension config installed + copy: + content: "{{ matrix_dimension_configuration|to_nice_yaml }}" + dest: "{{ matrix_dimension_base_path }}/config.yaml" + mode: 0640 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_dimension_user_gid }}" + +- name: Ensure Dimension image is pulled + docker_image: + name: "{{ matrix_dimension_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_dimension_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_dimension_docker_image_force_pull }}" + +- name: Ensure matrix-dimension.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-dimension.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-dimension.service" + mode: 0644 + register: matrix_dimension_systemd_service_result + +- name: Ensure systemd reloaded after matrix-dimension.service installation + service: + daemon_reload: yes + when: "matrix_dimension_systemd_service_result.changed|bool" + +- name: Ensure matrix-dimension.service restarted, if necessary + service: + name: "matrix-dimension.service" + state: restarted + when: "matrix_dimension_requires_restart|bool" diff --git a/roles/matrix-dimension/tasks/setup_uninstall.yml b/roles/matrix-dimension/tasks/setup_uninstall.yml new file mode 100644 index 00000000000..9bc4ac8b9e5 --- /dev/null +++ b/roles/matrix-dimension/tasks/setup_uninstall.yml @@ -0,0 +1,35 @@ +--- + +- name: Check existence of matrix-dimension service + stat: + path: "{{ matrix_systemd_path }}/matrix-dimension.service" + register: matrix_dimension_service_stat + +- name: Ensure matrix-dimension is stopped + service: + name: matrix-dimension + state: stopped + daemon_reload: yes + register: stopping_result + when: "matrix_dimension_service_stat.stat.exists|bool" + +- name: Ensure matrix-dimension.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-dimension.service" + state: absent + when: "matrix_dimension_service_stat.stat.exists|bool" + +- name: Ensure systemd reloaded after matrix-dimension.service removal + service: + daemon_reload: yes + when: "matrix_dimension_service_stat.stat.exists|bool" + +- name: Ensure Dimension base directory doesn't exist + file: + path: "{{ matrix_dimension_base_path }}" + state: absent + +- name: Ensure Dimension Docker image doesn't exist + docker_image: + name: "{{ matrix_dimension_docker_image }}" + state: absent diff --git a/roles/matrix-dimension/templates/config.yaml.j2 b/roles/matrix-dimension/templates/config.yaml.j2 index a05b6c35a63..200871e7e19 100644 --- a/roles/matrix-dimension/templates/config.yaml.j2 +++ b/roles/matrix-dimension/templates/config.yaml.j2 @@ -44,7 +44,11 @@ widgetBlacklist: # Where the database for Dimension is database: - file: "dimension.db" +{% if matrix_dimension_database_engine == 'sqlite' %} + file: {{ matrix_dimension_sqlite_database_path_in_container|to_json }} +{% elif matrix_dimension_database_engine == 'postgres' %} + uri: {{ matrix_dimension_database_connection_string|to_json }} +{% endif %} # Display settings that apply to self-hosted go-neb instances goneb: diff --git a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 index 94c38491059..ff10224af71 100644 --- a/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 +++ b/roles/matrix-dimension/templates/systemd/matrix-dimension.service.j2 @@ -1,8 +1,13 @@ #jinja2: lstrip_blocks: "True" [Unit] Description=Matrix Dimension -After=docker.service -Requires=docker.service +{% for service in matrix_dimension_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_dimension_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} DefaultDependencies=no [Service] @@ -11,7 +16,9 @@ ExecStartPre=-{{ matrix_host_command_docker }} kill matrix-dimension ExecStartPre=-{{ matrix_host_command_docker }} rm matrix-dimension # Fixup database ownership if it got changed somehow (during a server migration, etc.) -ExecStartPre=-{{ matrix_host_command_chown }} {{ matrix_dimension_user_uid }}:{{ matrix_dimension_user_gid }} {{ matrix_dimension_base_path }}/dimension.db +{% if matrix_dimension_database_engine == 'sqlite' %} +ExecStartPre=-{{ matrix_host_command_chown }} {{ matrix_dimension_user_uid }}:{{ matrix_dimension_user_gid }} {{ matrix_dimension_sqlite_database_path_local }} +{% endif %} ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-dimension \ --log-driver=none \ diff --git a/roles/matrix-ma1sd/defaults/main.yml b/roles/matrix-ma1sd/defaults/main.yml index f7a0782d5fb..2932f3ed422 100644 --- a/roles/matrix-ma1sd/defaults/main.yml +++ b/roles/matrix-ma1sd/defaults/main.yml @@ -39,6 +39,28 @@ matrix_ma1sd_systemd_wanted_services_list: [] # Enabling this is discouraged. Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/features/identity.md#lookups matrix_ma1sd_matrixorg_forwarding_enabled: false + +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_ma1sd_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_ma1sd_postgres_*` variables +matrix_ma1sd_database_engine: 'sqlite' + +matrix_ma1sd_sqlite_database_path_local: "{{ matrix_ma1sd_data_path }}/ma1sd.db" +matrix_ma1sd_sqlite_database_path_in_container: "/var/ma1sd/ma1sd.db" + +matrix_ma1sd_database_username: 'matrix_ma1sd' +matrix_ma1sd_database_password: 'some-password' +matrix_ma1sd_database_hostname: 'matrix-postgres' +matrix_ma1sd_database_port: 5432 +matrix_ma1sd_database_name: 'matrix_ma1sd' + +matrix_ma1sd_database_connection_string: 'postgresql://{{ matrix_ma1sd_database_username }}:{{ matrix_ma1sd_database_password }}@{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_name }}' + + # ma1sd has serveral supported identity stores. # One of them is storing identities directly in Synapse's database. # Learn more here: https://github.com/ma1uta/ma1sd/blob/master/docs/stores/synapse.md diff --git a/roles/matrix-ma1sd/tasks/main.yml b/roles/matrix-ma1sd/tasks/main.yml index f5ac34d650e..0b8a114e1de 100644 --- a/roles/matrix-ma1sd/tasks/main.yml +++ b/roles/matrix-ma1sd/tasks/main.yml @@ -8,7 +8,14 @@ - setup-all - setup-ma1sd -- import_tasks: "{{ role_path }}/tasks/setup_ma1sd.yml" +- import_tasks: "{{ role_path }}/tasks/setup_install.yml" + when: "run_setup|bool and matrix_ma1sd_enabled|bool" + tags: + - setup-all + - setup-ma1sd + +- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" + when: "run_setup|bool and not matrix_ma1sd_enabled|bool" tags: - setup-all - setup-ma1sd diff --git a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml b/roles/matrix-ma1sd/tasks/setup_install.yml similarity index 69% rename from roles/matrix-ma1sd/tasks/setup_ma1sd.yml rename to roles/matrix-ma1sd/tasks/setup_install.yml index 46acb42867e..9ae5f07798f 100644 --- a/roles/matrix-ma1sd/tasks/setup_ma1sd.yml +++ b/roles/matrix-ma1sd/tasks/setup_install.yml @@ -1,9 +1,5 @@ --- -# -# Tasks related to setting up ma1sd -# - - name: Ensure ma1sd paths exist file: path: "{{ item.path }}" @@ -15,10 +11,39 @@ - { path: "{{ matrix_ma1sd_config_path }}", when: true } - { path: "{{ matrix_ma1sd_data_path }}", when: true } - { path: "{{ matrix_ma1sd_docker_src_files_path }}", when: "{{ matrix_ma1sd_container_image_self_build }}"} - when: matrix_ma1sd_enabled|bool and item.when + when: "item.when|bool" - import_tasks: "{{ role_path }}/tasks/migrate_mxisd.yml" - when: matrix_ma1sd_enabled|bool + + +# These (SQLite -> Postgres) migration tasks are usually at the top, +# but we'd like to run them after `migrate_mxisd.yml`, which requires the ma1sd paths to exist. +- set_fact: + matrix_ma1sd_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_ma1sd_sqlite_database_path_local }}" + register: matrix_ma1sd_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_ma1sd_sqlite_database_path_local }}" + dst: "{{ matrix_ma1sd_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_ma1sd_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-ma1sd.service'] + pgloader_options: ['--with "quote identifiers"'] + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_ma1sd_requires_restart: true + when: "matrix_ma1sd_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_ma1sd_database_engine == 'postgres'" - name: Ensure ma1sd image is pulled docker_image: @@ -26,7 +51,7 @@ source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_ma1sd_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_ma1sd_docker_image_force_pull }}" - when: matrix_ma1sd_enabled|bool and not matrix_ma1sd_container_image_self_build + when: "not matrix_ma1sd_container_image_self_build|bool" - block: - name: Ensure gradle is installed for self-building (Debian) @@ -72,7 +97,7 @@ repository: "{{ matrix_ma1sd_docker_image }}" force_tag: yes source: local - when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_container_image_self_build" + when: "matrix_ma1sd_container_image_self_build|bool" - name: Ensure ma1sd config installed copy: @@ -81,7 +106,6 @@ mode: 0644 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" - when: matrix_ma1sd_enabled|bool - name: Ensure custom templates are installed if any copy: @@ -95,7 +119,7 @@ - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_session_validation_template }}", location: 'validate-template.eml'} - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_unbind_fraudulent_template }}", location: 'unbind-fraudulent.eml'} - {value: "{{ matrix_ma1sd_threepid_medium_email_custom_matrixid_template }}", location: 'mxid-template.eml'} - when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value" + when: "matrix_ma1sd_threepid_medium_email_custom_templates_enabled|bool and item.value" - name: Ensure matrix-ma1sd.service installed template: @@ -103,49 +127,14 @@ dest: "{{ matrix_systemd_path }}/matrix-ma1sd.service" mode: 0644 register: matrix_ma1sd_systemd_service_result - when: matrix_ma1sd_enabled|bool - name: Ensure systemd reloaded after matrix-ma1sd.service installation service: daemon_reload: yes - when: "matrix_ma1sd_enabled|bool and matrix_ma1sd_systemd_service_result.changed" - -# -# Tasks related to getting rid of ma1sd (if it was previously enabled) -# - -- name: Check existence of matrix-ma1sd service - stat: - path: "{{ matrix_systemd_path }}/matrix-ma1sd.service" - register: matrix_ma1sd_service_stat - -- name: Ensure matrix-ma1sd is stopped - service: - name: matrix-ma1sd - state: stopped - daemon_reload: yes - register: stopping_result - when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists" - -- name: Ensure matrix-ma1sd.service doesn't exist - file: - path: "{{ matrix_systemd_path }}/matrix-ma1sd.service" - state: absent - when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists" + when: "matrix_ma1sd_systemd_service_result.changed|bool" -- name: Ensure systemd reloaded after matrix-ma1sd.service removal +- name: Ensure matrix-ma1sd.service restarted, if necessary service: - daemon_reload: yes - when: "not matrix_ma1sd_enabled|bool and matrix_ma1sd_service_stat.stat.exists" - -- name: Ensure Matrix ma1sd paths don't exist - file: - path: "{{ matrix_ma1sd_base_path }}" - state: absent - when: "not matrix_ma1sd_enabled|bool" - -- name: Ensure ma1sd Docker image doesn't exist - docker_image: - name: "{{ matrix_ma1sd_docker_image }}" - state: absent - when: "not matrix_ma1sd_enabled|bool" + name: "matrix-ma1sd.service" + state: restarted + when: "matrix_ma1sd_requires_restart|bool" diff --git a/roles/matrix-ma1sd/tasks/setup_uninstall.yml b/roles/matrix-ma1sd/tasks/setup_uninstall.yml new file mode 100644 index 00000000000..b36ab508f10 --- /dev/null +++ b/roles/matrix-ma1sd/tasks/setup_uninstall.yml @@ -0,0 +1,35 @@ +--- + +- name: Check existence of matrix-ma1sd service + stat: + path: "{{ matrix_systemd_path }}/matrix-ma1sd.service" + register: matrix_ma1sd_service_stat + +- name: Ensure matrix-ma1sd is stopped + service: + name: matrix-ma1sd + state: stopped + daemon_reload: yes + register: stopping_result + when: "matrix_ma1sd_service_stat.stat.exists|bool" + +- name: Ensure matrix-ma1sd.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-ma1sd.service" + state: absent + when: "matrix_ma1sd_service_stat.stat.exists|bool" + +- name: Ensure systemd reloaded after matrix-ma1sd.service removal + service: + daemon_reload: yes + when: "matrix_ma1sd_service_stat.stat.exists|bool" + +- name: Ensure Matrix ma1sd paths don't exist + file: + path: "{{ matrix_ma1sd_base_path }}" + state: absent + +- name: Ensure ma1sd Docker image doesn't exist + docker_image: + name: "{{ matrix_ma1sd_docker_image }}" + state: absent diff --git a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 index 84585707413..9a426c477f1 100644 --- a/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 +++ b/roles/matrix-ma1sd/templates/ma1sd.yaml.j2 @@ -11,9 +11,19 @@ key: path: /var/ma1sd/sign.key storage: - provider: - sqlite: - database: /var/ma1sd/ma1sd.db + {% if matrix_ma1sd_database_engine == 'sqlite' %} + backend: sqlite + provider: + sqlite: + database: {{ matrix_ma1sd_sqlite_database_path_in_container|to_json }} + {% elif matrix_ma1sd_database_engine == 'postgres' %} + backend: postgresql + provider: + postgresql: + database: //{{ matrix_ma1sd_database_hostname }}:{{ matrix_ma1sd_database_port }}/{{ matrix_ma1sd_database_name }} + username: {{ matrix_ma1sd_database_username|to_json }} + password: {{ matrix_ma1sd_database_password|to_json }} + {% endif %} {% if matrix_ma1sd_dns_overwrite_enabled %} dns: diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index ec5cb3dca4f..8f1d0d78813 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -30,3 +30,50 @@ matrix_postgres_container_extra_arguments: [] # # Takes an ":" or "" value (e.g. "127.0.0.1:5432"), or empty string to not expose. matrix_postgres_container_postgres_bind_port: "" + +# A list of additional (databases and their credentials) to create. +# +# Example: +# matrix_postgres_additional_databases: +# - name: matrix_appservice_discord +# username: matrix_appservice_discord +# password: some_password +# - name: matrix_appservice_slack +# username: matrix_appservice_slack +# password: some_password +matrix_postgres_additional_databases: [] + +# A list of roles/users to avoid creating when importing (or upgrading) the database. +# If a dump file contains the roles and they've also been created beforehand (see `matrix_postgres_additional_databases`), +# importing would fail. +# We either need to not create them or to ignore the `CREATE ROLE` statements in the dump. +matrix_postgres_import_roles_to_ignore: [matrix_postgres_connection_username] + +matrix_postgres_import_roles_ignore_regex: "^CREATE ROLE ({{ matrix_postgres_import_roles_to_ignore|join('|') }});" + +# A list of databases to avoid creating when importing (or upgrading) the database. +# If a dump file contains the databases and they've also been created beforehand (see `matrix_postgres_additional_databases`), +# importing would fail. +# We either need to not create them or to ignore the `CREATE DATABASE` statements in the dump. +matrix_postgres_import_databases_to_ignore: [matrix_postgres_db_name] + +matrix_postgres_import_databases_ignore_regex: "^CREATE DATABASE ({{ matrix_postgres_import_databases_to_ignore|join('|') }})\\s" + +# The number of seconds to wait after starting `matrix-postgres.service` +# and before trying to run queries for creating additional databases/users against it. +# +# For most (subsequent) runs, Postgres would already be running, so no waiting will be happening at all. +matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds: 15 + + +matrix_postgres_pgloader_container_image_self_build: false +matrix_postgres_pgloader_container_image_self_build_repo: "https://github.com/illagrenan/pgloader-docker.git" +matrix_postgres_pgloader_container_image_self_build_repo_branch: "v{{ matrix_postgres_pgloader_docker_image_tag }}" +matrix_postgres_pgloader_container_image_self_build_src_path: "{{ matrix_postgres_base_path }}/pgloader-container-src" + +# We use illagrenan/pgloader, instead of the more official dimitri/pgloader image, +# because the official one only provides a `latest` tag. +matrix_postgres_pgloader_docker_image: "{{ matrix_postgres_pgloader_docker_image_name_prefix }}illagrenan/pgloader:{{ matrix_postgres_pgloader_docker_image_tag }}" +matrix_postgres_pgloader_docker_image_name_prefix: "{{ 'localhost/' if matrix_postgres_pgloader_container_image_self_build else 'docker.io/' }}" +matrix_postgres_pgloader_docker_image_tag: "3.6.2" +matrix_postgres_pgloader_docker_image_force_pull: "{{ matrix_postgres_pgloader_docker_image.endswith(':latest') }}" diff --git a/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml new file mode 100644 index 00000000000..a42c6f552ce --- /dev/null +++ b/roles/matrix-postgres/tasks/import_generic_sqlite_db.yml @@ -0,0 +1,97 @@ +--- + +# Pre-checks + +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot import." + when: "not matrix_postgres_enabled|bool" + +- name: Fail if playbook called incorrectly + fail: + msg: "The `sqlite_database_path` variable needs to be provided to this playbook, via --extra-vars" + when: "sqlite_database_path is not defined or sqlite_database_path.startswith('<')" + +- name: Check if the provided SQLite database file exists + stat: + path: "{{ sqlite_database_path }}" + register: sqlite_database_path_stat_result + +- name: Fail if provided SQLite database file doesn't exist + fail: + msg: "File cannot be found on the server at {{ sqlite_database_path }}" + when: "not sqlite_database_path_stat_result.stat.exists" + +# We either expect `postgres_db_connection_string` specifying a full Postgres database connection string, +# or `postgres_connection_string_variable_name`, specifying a name of a variable, which contains a valid connection string. + +- block: + - name: Fail if postgres_connection_string_variable_name points to an undefined variable + fail: msg="postgres_connection_string_variable_name is defined, but there is no variable with the name `{{ postgres_connection_string_variable_name }}`" + when: "postgres_connection_string_variable_name not in vars" + + - name: Get Postgres connection string from variable + set_fact: + postgres_db_connection_string: "{{ lookup('vars', postgres_connection_string_variable_name) }}" + when: 'postgres_connection_string_variable_name is defined' + +- name: Fail if playbook called incorrectly + fail: + msg: >- + Either a `postgres_db_connection_string` variable or a `postgres_connection_string_variable_name` needs to be provided to this playbook, via `--extra-vars`. + Example: `--extra-vars="postgres_db_connection_string=postgresql://username:password@localhost:/database_name"` or `--extra-vars="postgres_connection_string_variable_name=matrix_appservice_discord_database_connString"` + when: "postgres_db_connection_string is not defined or not postgres_db_connection_string.startswith('postgresql://')" + + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + + +# Actual import work + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +- name: Import SQLite database from {{ sqlite_database_path }} into Postgres + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ sqlite_database_path }},dst=/in.db,ro + --entrypoint=/bin/sh + {{ matrix_postgres_pgloader_docker_image }} + -c + 'pgloader /in.db {{ postgres_db_connection_string }}' + +- name: Archive SQLite database ({{ sqlite_database_path }} -> {{ sqlite_database_path }}.backup) + command: + cmd: "mv {{ sqlite_database_path }} {{ sqlite_database_path }}.backup" + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your SQLite database file has been imported into Postgres. The original file has been moved from `{{ sqlite_database_path }}` to `{{ sqlite_database_path }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file." + ] + }} diff --git a/roles/matrix-postgres/tasks/import_postgres.yml b/roles/matrix-postgres/tasks/import_postgres.yml index 33d98691dbe..c26affbbb9f 100644 --- a/roles/matrix-postgres/tasks/import_postgres.yml +++ b/roles/matrix-postgres/tasks/import_postgres.yml @@ -74,8 +74,8 @@ {{ matrix_postgres_docker_image_latest }} -c "cat /{{ server_path_postgres_dump|basename }} | {{ 'gunzip |' if server_path_postgres_dump.endswith('.gz') else '' }} - grep -vE '^CREATE ROLE {{ matrix_postgres_connection_username }}' | - grep -vE '^CREATE DATABASE {{ matrix_postgres_db_name }}' | + grep -vE '{{ matrix_postgres_import_roles_ignore_regex }}' | + grep -vE '{{ matrix_postgres_import_databases_ignore_regex }}' | psql -v ON_ERROR_STOP=1 -h matrix-postgres" # This is a hack. diff --git a/roles/matrix-postgres/tasks/main.yml b/roles/matrix-postgres/tasks/main.yml index f4c752a0401..b9c2ae7c99c 100644 --- a/roles/matrix-postgres/tasks/main.yml +++ b/roles/matrix-postgres/tasks/main.yml @@ -26,6 +26,12 @@ tags: - import-synapse-sqlite-db +# Perhaps we need a new variable here, instead of `run_postgres_import_sqlite_db`. +- import_tasks: "{{ role_path }}/tasks/import_generic_sqlite_db.yml" + when: run_postgres_import_sqlite_db|bool + tags: + - import-generic-sqlite-db + - import_tasks: "{{ role_path }}/tasks/upgrade_postgres.yml" when: run_postgres_upgrade|bool tags: diff --git a/roles/matrix-postgres/tasks/setup_postgres.yml b/roles/matrix-postgres/tasks/setup_postgres.yml index f186bdca4c0..518d1a5f028 100644 --- a/roles/matrix-postgres/tasks/setup_postgres.yml +++ b/roles/matrix-postgres/tasks/setup_postgres.yml @@ -113,6 +113,13 @@ daemon_reload: yes when: "matrix_postgres_enabled|bool and matrix_postgres_systemd_service_result.changed" +- include_tasks: + file: "{{ role_path }}/tasks/util/create_additional_databases.yml" + apply: + tags: + - always + when: "matrix_postgres_enabled|bool and matrix_postgres_additional_databases|length > 0" + # # Tasks related to getting rid of the internal postgres server (if it was previously enabled) # diff --git a/roles/matrix-postgres/tasks/upgrade_postgres.yml b/roles/matrix-postgres/tasks/upgrade_postgres.yml index 72f327b3551..564265d85b3 100644 --- a/roles/matrix-postgres/tasks/upgrade_postgres.yml +++ b/roles/matrix-postgres/tasks/upgrade_postgres.yml @@ -135,8 +135,8 @@ {{ matrix_postgres_docker_image_latest }} -c "cat /in/{{ postgres_dump_name }} | {{ 'gunzip |' if postgres_dump_name.endswith('.gz') else '' }} - grep -vE '^CREATE ROLE {{ matrix_postgres_connection_username }}' | - grep -vE '^CREATE DATABASE {{ matrix_postgres_db_name }}' | + grep -vE '{{ matrix_postgres_import_roles_ignore_regex }}' | + grep -vE '{{ matrix_postgres_import_databases_ignore_regex }}' | psql -v ON_ERROR_STOP=1 -h matrix-postgres" # This is a hack. diff --git a/roles/matrix-postgres/tasks/util/create_additional_database.yml b/roles/matrix-postgres/tasks/util/create_additional_database.yml new file mode 100644 index 00000000000..ce064d598e2 --- /dev/null +++ b/roles/matrix-postgres/tasks/util/create_additional_database.yml @@ -0,0 +1,40 @@ +--- + +# It'd be better if this is belonged to `validate_config.yml`, but it would have to be some loop-within-a-loop there, +# and that's ugly. We also don't expect this to catch errors often. It's more of a defensive last-minute check. +- name: Fail if additional database data appears invalid + fail: + msg: "Additional database definition ({{ additional_db }} lacks a required key: {{ item }}" + when: "item not in additional_db" + with_items: "{{ ['name', 'username', 'password'] }}" + +# The SQL statements that we'll run against Postgres are stored in a file that others can't read. +# This file will be mounted into the container and fed to Postgres. +# This way, we avoid passing sensitive data around in CLI commands that other users on the system can see. +- name: Create additional database initialization SQL file for {{ additional_db.name }} + template: + src: "{{ role_path }}/templates/init-additional-db-user-and-role.sql.j2" + dest: "/tmp/matrix-postgres-init-additional-db-user-and-role.sql" + mode: 0600 + owner: "{{ matrix_user_uid }}" + group: "{{ matrix_user_gid }}" + +- name: Execute Postgres additional database initialization SQL file for {{ additional_db.name }} + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --env-file={{ matrix_postgres_base_path }}/env-postgres-psql + --network {{ matrix_docker_network }} + --mount type=bind,src=/tmp/matrix-postgres-init-additional-db-user-and-role.sql,dst=/matrix-postgres-init-additional-db-user-and-role.sql,ro + --entrypoint=/bin/sh + {{ matrix_postgres_docker_image_to_use }} + -c + 'psql -h {{ matrix_postgres_connection_hostname }} --file=/matrix-postgres-init-additional-db-user-and-role.sql' + +- name: Delete additional database initialization SQL file for {{ additional_db.name }} + file: + path: /tmp/matrix-postgres-init-additional-db-user-and-role.sql + state: absent diff --git a/roles/matrix-postgres/tasks/util/create_additional_databases.yml b/roles/matrix-postgres/tasks/util/create_additional_databases.yml new file mode 100644 index 00000000000..0ad460dddb9 --- /dev/null +++ b/roles/matrix-postgres/tasks/util/create_additional_databases.yml @@ -0,0 +1,23 @@ +--- + +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ matrix_postgres_additional_databases_postgres_start_wait_timeout_seconds }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +- name: Create additional Postgres user and database + include_tasks: "{{ role_path }}/tasks/util/create_additional_database.yml" + with_items: "{{ matrix_postgres_additional_databases }}" + loop_control: + loop_var: additional_db + # Suppress logging to avoid dumping the credentials to the shell + no_log: true diff --git a/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml new file mode 100644 index 00000000000..0da48c64eb4 --- /dev/null +++ b/roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml @@ -0,0 +1,167 @@ +--- + +- name: Fail if Postgres not enabled + fail: + msg: "Postgres via the matrix-postgres role is not enabled (`matrix_postgres_enabled`). Cannot migrate." + when: "not matrix_postgres_enabled|bool" + +- name: Fail if util called incorrectly (missing matrix_postgres_db_migration_request) + fail: + msg: "The `matrix_postgres_db_migration_request` variable needs to be provided to this util." + when: "matrix_postgres_db_migration_request is not defined" + +- name: Fail if util called incorrectly (invalid matrix_postgres_db_migration_request) + fail: + msg: "The `matrix_postgres_db_migration_request` variable needs to contain `{{ item }}`." + with_items: + - src + - dst + - caller + - engine_variable_name + - systemd_services_to_stop + when: "item not in matrix_postgres_db_migration_request" + +- name: Check if the provided source database file exists + stat: + path: "{{ matrix_postgres_db_migration_request.src }}" + register: matrix_postgres_db_migration_request_src_stat_result + +- name: Fail if provided source database file doesn't exist + fail: + msg: "File cannot be found on the server at {{ matrix_postgres_db_migration_request.src }}" + when: "not matrix_postgres_db_migration_request_src_stat_result.stat.exists" + +- block: + - name: Ensure pgloader repository is present on self-build + git: + repo: "{{ matrix_postgres_pgloader_container_image_self_build_repo }}" + dest: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}" + version: "{{ matrix_postgres_pgloader_container_image_self_build_repo_branch }}" + force: "yes" + register: matrix_postgres_pgloader_git_pull_results + + # If `stable` is used, we hit an error when processing /opt/src/pgloader/build/quicklisp/dists/quicklisp/software/uax-15-20201220-git/data/CompositionExclusions.txt: + # > the octet sequence #(194) cannot be decoded + # + # The issue is described here and is not getting fixed for months: https://github.com/dimitri/pgloader/pull/1179 + # + # Although we're not using the dimitri/pgloader image, the one we're using suffers from the same problem. + - name: Switch pgloader base image from Debian stable (likely 10.x/Buster) to Bullseye + lineinfile: + path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}/Dockerfile" + regexp: "{{ item.match }}" + line: "{{ item.replace }}" + with_items: + - match: '^FROM debian:stable-slim as builder$' + replace: 'FROM debian:bullseye-slim as builder' + - match: '^FROM debian:stable-slim$' + replace: 'FROM debian:bullseye-slim' + + - name: Ensure pgloader Docker image is built + docker_image: + name: "{{ matrix_postgres_pgloader_docker_image }}" + source: build + force_source: "{{ matrix_postgres_pgloader_git_pull_results.changed }}" + build: + dockerfile: Dockerfile + path: "{{ matrix_postgres_pgloader_container_image_self_build_src_path }}" + pull: yes + when: "matrix_postgres_pgloader_container_image_self_build|bool" + +- name: Ensure pgloader Docker image is pulled + docker_image: + name: "{{ matrix_postgres_pgloader_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_postgres_pgloader_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_postgres_pgloader_docker_image_force_pull }}" + when: "not matrix_postgres_pgloader_container_image_self_build" + +# Defaults + +- name: Set postgres_start_wait_time, if not provided + set_fact: + postgres_start_wait_time: 15 + when: "postgres_start_wait_time|default('') == ''" + +# Actual import work + +# matrix-postgres is most likely started already +- name: Ensure matrix-postgres is started + service: + name: matrix-postgres + state: started + daemon_reload: yes + register: matrix_postgres_service_start_result + +- name: Wait a bit, so that Postgres can start + wait_for: + timeout: "{{ postgres_start_wait_time }}" + delegate_to: 127.0.0.1 + become: false + when: "matrix_postgres_service_start_result.changed|bool" + +# We only stop services here, leaving it to the caller to start them later. +# +# We can't start them, because they probably need to be reconfigured too (changing the configuration from using SQLite to Postgres, etc.), +# before starting. +# +# Since the caller will be starting them, it might make sense to leave stopping to it as well. +# However, we don't do it, because it's simpler having it here, and it also gets to happen only if we'll be doing an import. +# If we bailed out (somewhere above), nothing would have gotten stopped. It's nice to leave this running in such cases. +- name: Ensure systemd services blocking the database import are stopped + service: + name: "{{ item }}" + state: stopped + with_items: "{{ matrix_postgres_db_migration_request.systemd_services_to_stop }}" + +- name: Import {{ matrix_postgres_db_migration_request.engine_old }} database from {{ matrix_postgres_db_migration_request.src }} into Postgres + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --network={{ matrix_docker_network }} + --mount type=bind,src={{ matrix_postgres_db_migration_request.src }},dst=/in.db,ro + --entrypoint=/bin/sh + {{ matrix_postgres_pgloader_docker_image }} + -c + 'pgloader {{ matrix_postgres_db_migration_request.pgloader_options|default([])|join(' ') }} /in.db {{ matrix_postgres_db_migration_request.dst }}' + +- name: Archive {{ matrix_postgres_db_migration_request.engine_old }} database ({{ matrix_postgres_db_migration_request.src }} -> {{ matrix_postgres_db_migration_request.src }}.backup) + command: + cmd: "mv {{ matrix_postgres_db_migration_request.src }} {{ matrix_postgres_db_migration_request.src }}.backup" + +- block: + # We can't use `{{ role_path }}` here, neither with `import_tasks`, nor with `include_tasks`, + # because it refers to the role that included this util, and not to the role this file belongs to. + - import_tasks: "roles/matrix-postgres/tasks/util/detect_existing_postgres_version.yml" + + - set_fact: + matrix_postgres_docker_image_to_use: "{{ matrix_postgres_docker_image_latest if matrix_postgres_detected_version_corresponding_docker_image == '' else matrix_postgres_detected_version_corresponding_docker_image }}" + + - name: Execute additional Postgres SQL migration statements + command: + cmd: >- + {{ matrix_host_command_docker }} run + --rm + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} + --cap-drop=ALL + --env-file={{ matrix_postgres_base_path }}/env-postgres-psql + --network={{ matrix_docker_network }} + {{ matrix_postgres_docker_image_to_use }} + psql --host=matrix-postgres --dbname={{ matrix_postgres_db_migration_request.additional_psql_statements_db_name }} --command='{{ item }}' + with_items: "{{ matrix_postgres_db_migration_request.additional_psql_statements_list }}" + + when: "matrix_postgres_db_migration_request.additional_psql_statements_list|default([])|length > 0" + +- name: Inject result + set_fact: + matrix_playbook_runtime_results: | + {{ + matrix_playbook_runtime_results|default([]) + + + [ + "NOTE: Your {{ matrix_postgres_db_migration_request.engine_old }} database file has been imported into Postgres. The original database file has been moved from `{{ matrix_postgres_db_migration_request.src }}` to `{{ matrix_postgres_db_migration_request.src }}.backup`. When you've confirmed that the import went well and everything works, you should be able to safely delete this file." + ] + }} diff --git a/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 new file mode 100644 index 00000000000..609a1344824 --- /dev/null +++ b/roles/matrix-postgres/templates/init-additional-db-user-and-role.sql.j2 @@ -0,0 +1,19 @@ +-- `CREATE USER` does not support `IF NOT EXISTS`, so we use this workaround to prevent an error and raise a notice instead. +-- Seen here: https://stackoverflow.com/a/49858797 +DO $$ +BEGIN + CREATE USER {{ additional_db.username }}; + EXCEPTION WHEN DUPLICATE_OBJECT THEN + RAISE NOTICE 'not creating user {{ additional_db.username }}, since it already exists'; +END +$$; + +-- This is useful for initial user creation (since we don't assign a password above) and for handling subsequent password changes +-- TODO - we should escape quotes in the password. +ALTER ROLE {{ additional_db.username }} PASSWORD '{{ additional_db.password }}'; + +-- This will generate an error on subsequent execution +CREATE DATABASE {{ additional_db.name }} WITH LC_CTYPE 'C' LC_COLLATE 'C' OWNER {{ additional_db.username }}; + +-- This is useful for changing the database owner subsequently +ALTER DATABASE {{ additional_db.name }} OWNER TO {{ additional_db.username }}; diff --git a/roles/matrix-registration/defaults/main.yml b/roles/matrix-registration/defaults/main.yml index d89bf070dbf..065e0c482a4 100644 --- a/roles/matrix-registration/defaults/main.yml +++ b/roles/matrix-registration/defaults/main.yml @@ -32,6 +32,36 @@ matrix_registration_systemd_wanted_services_list: [] # Takes an ":" or "" value (e.g. "127.0.0.1:8767"), or empty string to not expose. matrix_registration_container_http_host_bind_port: '' +# Database-related configuration fields. +# +# To use SQLite, stick to these defaults. +# +# To use Postgres: +# - change the engine (`matrix_registration_database_engine: 'postgres'`) +# - adjust your database credentials via the `matrix_registration_postgres_*` variables +matrix_registration_database_engine: 'sqlite' + +matrix_registration_sqlite_database_path_local: "{{ matrix_registration_data_path }}/db.sqlite3" +matrix_registration_sqlite_database_path_in_container: "/data/db.sqlite3" + +matrix_registration_database_username: 'matrix_registration' +matrix_registration_database_password: 'some-password' +matrix_registration_database_hostname: 'matrix-postgres' +matrix_registration_database_port: 5432 +matrix_registration_database_name: 'matrix_registration' + +matrix_registration_database_connection_string: 'postgresql://{{ matrix_registration_database_username }}:{{ matrix_registration_database_password }}@{{ matrix_registration_database_hostname }}:{{ matrix_registration_database_port }}/{{ matrix_registration_database_name }}' + +# For some reason, matrix-registraiton expects the `db` field to be like this: `sqlite:////data/db.sqlite3`. +# (seems like one too many slashes, but..) +matrix_registration_db: "{{ + { + 'sqlite': ('sqlite:///' + matrix_registration_sqlite_database_path_in_container), + 'postgres': matrix_registration_database_connection_string, + }[matrix_registration_database_engine] +}}" + + # The path at which Matrix Registration will be exposed on `matrix.DOMAIN` # (only applies when matrix-nginx-proxy is used). matrix_registration_public_endpoint: /matrix-registration @@ -58,7 +88,6 @@ matrix_registration_admin_secret: "" matrix_registration_riot_instance: "https://riot.im/app/" - # Default matrix-registration configuration template which covers the generic use case. # You can customize it by controlling the various variables inside it. # diff --git a/roles/matrix-registration/tasks/main.yml b/roles/matrix-registration/tasks/main.yml index 4a884ccd9c2..4fef6abe97d 100644 --- a/roles/matrix-registration/tasks/main.yml +++ b/roles/matrix-registration/tasks/main.yml @@ -8,7 +8,14 @@ - setup-all - setup-matrix-registration -- import_tasks: "{{ role_path }}/tasks/setup.yml" +- import_tasks: "{{ role_path }}/tasks/setup_install.yml" + when: "run_setup|bool and matrix_registration_enabled|bool" + tags: + - setup-all + - setup-matrix-registration + +- import_tasks: "{{ role_path }}/tasks/setup_uninstall.yml" + when: "run_setup|bool and not matrix_registration_enabled|bool" tags: - setup-all - setup-matrix-registration diff --git a/roles/matrix-registration/tasks/setup.yml b/roles/matrix-registration/tasks/setup_install.yml similarity index 55% rename from roles/matrix-registration/tasks/setup.yml rename to roles/matrix-registration/tasks/setup_install.yml index dfe1ba8ba6c..9b6d126018d 100644 --- a/roles/matrix-registration/tasks/setup.yml +++ b/roles/matrix-registration/tasks/setup_install.yml @@ -1,8 +1,35 @@ --- -# -# Tasks related to setting up matrix-registration -# +- set_fact: + matrix_registration_requires_restart: false + +- block: + - name: Check if an SQLite database already exists + stat: + path: "{{ matrix_registration_sqlite_database_path_local }}" + register: matrix_registration_sqlite_database_path_local_stat_result + + - block: + - set_fact: + matrix_postgres_db_migration_request: + src: "{{ matrix_registration_sqlite_database_path_local }}" + dst: "{{ matrix_registration_database_connection_string }}" + caller: "{{ role_path|basename }}" + engine_variable_name: 'matrix_registration_database_engine' + engine_old: 'sqlite' + systemd_services_to_stop: ['matrix-registration.service'] + # pgloader makes `ex_date` of type `TIMESTAMP WITH TIMEZONE`, + # which makes matrix-registration choke on it later on when comparing dates. + additional_psql_statements_list: + - ALTER TABLE tokens ALTER COLUMN ex_date TYPE TIMESTAMP WITHOUT TIME ZONE; + additional_psql_statements_db_name: "{{ matrix_registration_database_name }}" + + - import_tasks: "roles/matrix-postgres/tasks/util/migrate_db_to_postgres.yml" + + - set_fact: + matrix_registration_requires_restart: true + when: "matrix_registration_sqlite_database_path_local_stat_result.stat.exists|bool" + when: "matrix_registration_database_engine == 'postgres'" - name: Ensure matrix-registration paths exist file: @@ -16,7 +43,7 @@ - { path: "{{ matrix_registration_config_path }}", when: true } - { path: "{{ matrix_registration_data_path }}", when: true } - { path: "{{ matrix_registration_docker_src_files_path }}", when: "{{ matrix_registration_container_image_self_build }}"} - when: matrix_registration_enabled|bool and item.when + when: "item.when|bool" - name: Ensure matrix-registration image is pulled docker_image: @@ -24,7 +51,7 @@ source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" force_source: "{{ matrix_registration_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_registration_docker_image_force_pull }}" - when: "matrix_registration_enabled|bool and not matrix_registration_container_image_self_build|bool" + when: "not matrix_registration_container_image_self_build|bool" - name: Ensure matrix-registration repository is present when self-building git: @@ -33,7 +60,7 @@ version: "{{ matrix_registration_container_image_self_build_branch }}" force: "yes" register: matrix_registration_git_pull_results - when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool" + when: "matrix_registration_container_image_self_build|bool" - name: Ensure matrix-registration Docker image is built docker_image: @@ -44,7 +71,7 @@ dockerfile: Dockerfile path: "{{ matrix_registration_docker_src_files_path }}" pull: yes - when: "matrix_registration_enabled|bool and matrix_registration_container_image_self_build|bool" + when: "matrix_registration_container_image_self_build|bool" - name: Ensure matrix-registration config installed copy: @@ -53,7 +80,6 @@ mode: 0644 owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" - when: matrix_registration_enabled|bool - name: Ensure matrix-registration.service installed template: @@ -61,43 +87,14 @@ dest: "{{ matrix_systemd_path }}/matrix-registration.service" mode: 0644 register: matrix_registration_systemd_service_result - when: matrix_registration_enabled|bool - name: Ensure systemd reloaded after matrix-registration.service installation service: daemon_reload: yes - when: "matrix_registration_enabled|bool and matrix_registration_systemd_service_result.changed" + when: "matrix_registration_systemd_service_result.changed|bool" -# -# Tasks related to getting rid of matrix-registration (if it was previously enabled) -# - -- name: Check existence of matrix-registration service - stat: - path: "{{ matrix_systemd_path }}/matrix-registration.service" - register: matrix_registration_service_stat - -- name: Ensure matrix-registration is stopped +- name: Ensure matrix-registration.service restarted, if necessary service: - name: matrix-registration - state: stopped - daemon_reload: yes - register: stopping_result - when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists" - -- name: Ensure matrix-registration.service doesn't exist - file: - path: "{{ matrix_systemd_path }}/matrix-registration.service" - state: absent - when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists" - -- name: Ensure systemd reloaded after matrix-registration.service removal - service: - daemon_reload: yes - when: "not matrix_registration_enabled|bool and matrix_registration_service_stat.stat.exists" - -- name: Ensure matrix-registration Docker image doesn't exist - docker_image: - name: "{{ matrix_registration_docker_image }}" - state: absent - when: "not matrix_registration_enabled|bool" + name: "matrix-registration.service" + state: restarted + when: "matrix_registration_requires_restart|bool" diff --git a/roles/matrix-registration/tasks/setup_uninstall.yml b/roles/matrix-registration/tasks/setup_uninstall.yml new file mode 100644 index 00000000000..573f8170b41 --- /dev/null +++ b/roles/matrix-registration/tasks/setup_uninstall.yml @@ -0,0 +1,30 @@ +--- + +- name: Check existence of matrix-registration service + stat: + path: "{{ matrix_systemd_path }}/matrix-registration.service" + register: matrix_registration_service_stat + +- name: Ensure matrix-registration is stopped + service: + name: matrix-registration + state: stopped + daemon_reload: yes + register: stopping_result + when: "matrix_registration_service_stat.stat.exists|bool" + +- name: Ensure matrix-registration.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-registration.service" + state: absent + when: "matrix_registration_service_stat.stat.exists|bool" + +- name: Ensure systemd reloaded after matrix-registration.service removal + service: + daemon_reload: yes + when: "matrix_registration_service_stat.stat.exists|bool" + +- name: Ensure matrix-registration Docker image doesn't exist + docker_image: + name: "{{ matrix_registration_docker_image }}" + state: absent diff --git a/roles/matrix-registration/templates/config.yaml.j2 b/roles/matrix-registration/templates/config.yaml.j2 index 1b2464e50a5..39211b249b5 100644 --- a/roles/matrix-registration/templates/config.yaml.j2 +++ b/roles/matrix-registration/templates/config.yaml.j2 @@ -3,7 +3,7 @@ server_name: {{ matrix_registration_server_name|to_json }} shared_secret: {{ matrix_registration_shared_secret|to_json }} admin_secret: {{ matrix_registration_admin_secret|to_json }} riot_instance: {{ matrix_registration_riot_instance|to_json }} -db: 'sqlite:////data/db.sqlite3' +db: {{ matrix_registration_db|to_json }} host: '0.0.0.0' port: 5000 rate_limit: ["100 per day", "10 per minute"]