diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index b82dbf23ae8..5c803a092ce 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -864,6 +864,22 @@ matrix_postgres_db_name: "homeserver" +###################################################################### +# +# matrix-redis +# +###################################################################### + +matrix_redis_enabled: "{{ matrix_synapse_workers_enabled }}" + +###################################################################### +# +# /matrix-redis +# +###################################################################### + + + ###################################################################### # # matrix-client-element @@ -1003,6 +1019,11 @@ matrix_synapse_systemd_wanted_services_list: | (['matrix-mailer.service'] if matrix_mailer_enabled else []) }} +# Worker support with redis +matrix_synapse_redis_enabled: "{{ matrix_redis_enabled }}" +matrix_synapse_redis_host: "{{ 'matrix-redis' if matrix_redis_enabled else '' }}" +matrix_synapse_redis_password: "{{ matrix_redis_connection_password if matrix_redis_enabled else '' }}" + ###################################################################### # # /matrix-synapse diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 8ba0c532afb..4ae22f238ea 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -247,3 +247,81 @@ matrix_ssl_log_dir_path: "{{ matrix_ssl_base_path }}/log" # nginx status page configurations. matrix_nginx_proxy_proxy_matrix_nginx_status_enabled: false matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: ['{{ ansible_default_ipv4.address }}'] + + +# worker +matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}" +matrix_nginx_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}" +matrix_nginx_proxy_synapse_generic_worker_locations: [ + # Sync requests + '^/_matrix/client/(v2_alpha|r0)/sync$', + '^/_matrix/client/(api/v1|v2_alpha|r0)/events$', + '^/_matrix/client/(api/v1|r0)/initialSync$', + '^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$', + + # Federation requests + '^/_matrix/federation/v1/event/', + '^/_matrix/federation/v1/state/', + '^/_matrix/federation/v1/state_ids/', + '^/_matrix/federation/v1/backfill/', + '^/_matrix/federation/v1/get_missing_events/', + '^/_matrix/federation/v1/publicRooms', + '^/_matrix/federation/v1/query/', + '^/_matrix/federation/v1/make_join/', + '^/_matrix/federation/v1/make_leave/', + '^/_matrix/federation/v1/send_join/', + '^/_matrix/federation/v2/send_join/', + '^/_matrix/federation/v1/send_leave/', + '^/_matrix/federation/v2/send_leave/', + '^/_matrix/federation/v1/invite/', + '^/_matrix/federation/v2/invite/', + '^/_matrix/federation/v1/query_auth/', + '^/_matrix/federation/v1/event_auth/', + '^/_matrix/federation/v1/exchange_third_party_invite/', + '^/_matrix/federation/v1/user/devices/', + '^/_matrix/federation/v1/get_groups_publicised$', + '^/_matrix/key/v2/query', + + # Inbound federation transaction request + '^/_matrix/federation/v1/send/', + + # Client API requests + '^/_matrix/client/(api/v1|r0|unstable)/publicRooms$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$', + '^/_matrix/client/(api/v1|r0|unstable)/account/3pid$', + '^/_matrix/client/(api/v1|r0|unstable)/keys/query$', + '^/_matrix/client/(api/v1|r0|unstable)/keys/changes$', + '^/_matrix/client/versions$', + '^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$', + '^/_matrix/client/(api/v1|r0|unstable)/joined_groups$', + '^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$', + '^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/', + + # Registration/login requests + '^/_matrix/client/(api/v1|r0|unstable)/login$', + '^/_matrix/client/(r0|unstable)/register$', + '^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$', + + # Event sending requests + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$', + '^/_matrix/client/(api/v1|r0|unstable)/join/', + '^/_matrix/client/(api/v1|r0|unstable)/profile/', +] + +matrix_nginx_proxy_synapse_media_repository_locations: [ + '^/_matrix/media/', + '^/_synapse/admin/v1/purge_media_cache$', + '^/_synapse/admin/v1/room/.*/media.*$', + '^/_synapse/admin/v1/user/.*/media.*$', + '^/_synapse/admin/v1/media/.*$', + '^/_synapse/admin/v1/quarantine_media/.*$', +] + +matrix_nginx_proxy_synapse_user_dir_locations: [ + '^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$', +] diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 8fd87958676..a49bd8b6be4 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -101,6 +101,61 @@ } {% endif %} + {% if matrix_nginx_proxy_synapse_workers_enabled %} + {# Synapse Workers #} + + {% if generic_worker_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #} + {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %} + location ~ {{ location }} { + proxy_pass http://generic_worker_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {# ToDo: add GET ^/_matrix/federation/v1/groups/ #} + {% endif %} + + {% if media_repository_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository #} + {% for location in matrix_nginx_proxy_synapse_media_repository_locations %} + location ~ {{ location }} { + proxy_pass http://media_repository_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {% endif %} + + {% if user_dir_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappuser_dir #} + {% for location in matrix_nginx_proxy_synapse_user_dir_locations %} + location ~ {{ location }} { + proxy_pass http://user_dir_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {% endif %} + + {% if frontend_proxy_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfrontend_proxy #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { + proxy_pass http://frontend_proxy_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% if not matrix_synapse_use_presence %} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { + proxy_pass http://frontend_proxy_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + {% endif %} + {% endif %} + + {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} {{- configuration_block }} {% endfor %} @@ -174,6 +229,53 @@ } {% endmacro %} +{% set generic_worker_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'generic_worker')|list %} +{% set media_repository_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %} +{% set user_dir_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %} +{% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|list %} +{% if matrix_nginx_proxy_synapse_workers_enabled %} + {# Setup upstream for groups of workers #} + + {% if generic_worker_workers %} + upstream generic_worker_upstream { + # ensures that requests from the same client will always be passed + # to the same server (except when this server is unavailable) + ip_hash; + + {% for worker in generic_worker_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} + + {% if frontend_proxy_workers %} + upstream frontend_proxy_upstream { + # Round Robin + {% for worker in frontend_proxy_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} + + {% if media_repository_workers %} + upstream media_repository_upstream { + # Round Robin + {% for worker in media_repository_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} + + {% if user_dir_workers %} + upstream user_dir_upstream { + # Round Robin + {% for worker in user_dir_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} +{% endif %} + server { listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }}; server_name {{ matrix_nginx_proxy_proxy_matrix_hostname }}; @@ -255,6 +357,20 @@ server { ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; {% endif %} + {% if matrix_nginx_proxy_synapse_workers_enabled %} + {% if generic_worker_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #} + {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %} + location ~ {{ location }} { + proxy_pass http://generic_worker_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {# ToDo: add GET ^/_matrix/federation/v1/groups/ #} + {% endif %} + {% endif %} + location / { {% if matrix_nginx_proxy_enabled %} {# Use the embedded DNS resolver in Docker containers to discover the service #} diff --git a/roles/matrix-postgres/defaults/main.yml b/roles/matrix-postgres/defaults/main.yml index b5c6f6a88ea..6b824a234af 100644 --- a/roles/matrix-postgres/defaults/main.yml +++ b/roles/matrix-postgres/defaults/main.yml @@ -22,6 +22,10 @@ matrix_postgres_docker_image_force_pull: "{{ matrix_postgres_docker_image_to_use # A list of extra arguments to pass to the container matrix_postgres_container_extra_arguments: [] +# A list of extra arguments to pass to the postgres process +# e.g. "-c 'max_connections=200'" +matrix_postgres_process_extra_arguments: [] + # Controls whether the matrix-postgres container exposes a port (tcp/5432 in the # container) that can be used to access the database from outside the container (e.g. with psql) # diff --git a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 index 8e6392c1a5b..f1c751a6a72 100644 --- a/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 +++ b/roles/matrix-postgres/templates/systemd/matrix-postgres.service.j2 @@ -26,7 +26,11 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-postgres \ {% for arg in matrix_postgres_container_extra_arguments %} {{ arg }} \ {% endfor %} - {{ matrix_postgres_docker_image_to_use }} + {{ matrix_postgres_docker_image_to_use }} \ + postgres \ + {% for arg in matrix_postgres_process_extra_arguments %} + {{ arg }} \ + {% endfor %} ExecStop=-{{ matrix_host_command_docker }} stop matrix-postgres ExecStop=-{{ matrix_host_command_docker }} rm matrix-postgres diff --git a/roles/matrix-redis/defaults/main.yml b/roles/matrix-redis/defaults/main.yml new file mode 100644 index 00000000000..f48ea542103 --- /dev/null +++ b/roles/matrix-redis/defaults/main.yml @@ -0,0 +1,22 @@ +matrix_redis_enabled: true + +matrix_redis_connection_password: "" + +matrix_redis_base_path: "{{ matrix_base_data_path }}/redis" +matrix_redis_data_path: "{{ matrix_redis_base_path }}/data" + +matrix_redis_docker_image_v5: "redis:5.0-alpine" +matrix_redis_docker_image_v6: "redis:6.0-alpine" +matrix_redis_docker_image_latest: "{{ matrix_redis_docker_image_v6 }}" +matrix_redis_docker_image_to_use: '{{ matrix_redis_docker_image_latest }}' + +matrix_redis_docker_image_force_pull: "{{ matrix_redis_docker_image_to_use.endswith(':latest') }}" + +# A list of extra arguments to pass to the container +matrix_redis_container_extra_arguments: [] + +# Controls whether the matrix-redis container exposes a port (tcp/6379 in the container) +# that can be used to access redis from outside the container +# +# Takes an ":" or "" value (e.g. "127.0.0.1:6379"), or empty string to not expose. +matrix_redis_container_redis_bind_port: "" diff --git a/roles/matrix-redis/tasks/init.yml b/roles/matrix-redis/tasks/init.yml new file mode 100644 index 00000000000..49068851282 --- /dev/null +++ b/roles/matrix-redis/tasks/init.yml @@ -0,0 +1,3 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-redis'] }}" + when: matrix_redis_enabled|bool diff --git a/roles/matrix-redis/tasks/main.yml b/roles/matrix-redis/tasks/main.yml new file mode 100644 index 00000000000..595b09f5511 --- /dev/null +++ b/roles/matrix-redis/tasks/main.yml @@ -0,0 +1,9 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/setup_redis.yml" + when: run_setup|bool + tags: + - setup-all + - setup-redis diff --git a/roles/matrix-redis/tasks/setup_redis.yml b/roles/matrix-redis/tasks/setup_redis.yml new file mode 100644 index 00000000000..6f00282b459 --- /dev/null +++ b/roles/matrix-redis/tasks/setup_redis.yml @@ -0,0 +1,99 @@ +--- + +# +# Tasks related to setting up an internal redis server +# + +- name: Ensure redis Docker image is pulled + docker_image: + name: "{{ matrix_redis_docker_image_to_use }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_redis_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_redis_docker_image_force_pull }}" + when: matrix_redis_enabled|bool + +- name: Ensure redis paths exist + file: + path: "{{ item }}" + state: directory + mode: 0700 + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_username }}" + with_items: + - "{{ matrix_redis_base_path }}" + - "{{ matrix_redis_data_path }}" + when: matrix_redis_enabled|bool + +# We do this as a separate task, because: +# - we'd like to do it for the data path only, not for the base path (which contains root-owned environment variable files we'd like to leave as-is) +# - we need to do it without `mode`, or we risk making certain `.conf` and other files's executable bit to flip to true +- name: Ensure redis data path ownership is correct + file: + path: "{{ matrix_redis_data_path }}" + state: directory + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_username }}" + recurse: yes + when: matrix_redis_enabled|bool + +- name: Ensure redis environment variables file created + template: + src: "{{ role_path }}/templates/{{ item }}.j2" + dest: "{{ matrix_redis_base_path }}/{{ item }}" + mode: 0644 + with_items: + - "redis.conf" + when: matrix_redis_enabled|bool + +- name: Ensure matrix-redis.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-redis.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-redis.service" + mode: 0644 + register: matrix_redis_systemd_service_result + when: matrix_redis_enabled|bool + +- name: Ensure systemd reloaded after matrix-redis.service installation + service: + daemon_reload: yes + when: "matrix_redis_enabled|bool and matrix_redis_systemd_service_result.changed" + +# +# Tasks related to getting rid of the internal redis server (if it was previously enabled) +# + +- name: Check existence of matrix-redis service + stat: + path: "{{ matrix_systemd_path }}/matrix-redis.service" + register: matrix_redis_service_stat + when: "not matrix_redis_enabled|bool" + +- name: Ensure matrix-redis is stopped + service: + name: matrix-redis + state: stopped + daemon_reload: yes + when: "not matrix_redis_enabled|bool and matrix_redis_service_stat.stat.exists" + +- name: Ensure matrix-redis.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-redis.service" + state: absent + when: "not matrix_redis_enabled|bool and matrix_redis_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-redis.service removal + service: + daemon_reload: yes + when: "not matrix_redis_enabled|bool and matrix_redis_service_stat.stat.exists" + +- name: Check existence of matrix-redis local data path + stat: + path: "{{ matrix_redis_data_path }}" + register: matrix_redis_data_path_stat + when: "not matrix_redis_enabled|bool" + +# We just want to notify the user. Deleting data is too destructive. +- name: Notify if matrix-redis local data remains + debug: + msg: "Note: You are not using a local redis instance, but some old data remains from before in `{{ matrix_redis_data_path }}`. Feel free to delete it." + when: "not matrix_redis_enabled|bool and matrix_redis_data_path_stat.stat.exists" diff --git a/roles/matrix-redis/templates/redis.conf.j2 b/roles/matrix-redis/templates/redis.conf.j2 new file mode 100644 index 00000000000..3437135663c --- /dev/null +++ b/roles/matrix-redis/templates/redis.conf.j2 @@ -0,0 +1,4 @@ +#jinja2: lstrip_blocks: "True" +{% if matrix_redis_connection_password %} +requirepass {{ matrix_redis_connection_password }} +{% endif %} diff --git a/roles/matrix-redis/templates/systemd/matrix-redis.service.j2 b/roles/matrix-redis/templates/systemd/matrix-redis.service.j2 new file mode 100644 index 00000000000..0752d23bad6 --- /dev/null +++ b/roles/matrix-redis/templates/systemd/matrix-redis.service.j2 @@ -0,0 +1,36 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=Matrix Redis server +After=docker.service +Requires=docker.service + +[Service] +Type=simple +ExecStartPre=-/usr/bin/docker stop matrix-redis +ExecStartPre=-/usr/bin/docker rm matrix-redis + +ExecStart=/usr/bin/docker run --rm --name matrix-redis \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --read-only \ + --tmpfs=/tmp:rw,noexec,nosuid,size=100m \ + --network={{ matrix_docker_network }} \ + {% if matrix_redis_container_redis_bind_port %} + -p {{ matrix_redis_container_redis_bind_port }}:6379 \ + {% endif %} + -v {{ matrix_redis_base_path }}/redis.conf:/usr/local/etc/redis/redis.conf \ + {% for arg in matrix_redis_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_redis_docker_image_to_use }} \ + redis-server /usr/local/etc/redis/redis.conf + +ExecStop=-/usr/bin/docker stop matrix-redis +ExecStop=-/usr/bin/docker rm matrix-redis +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-redis + +[Install] +WantedBy=multi-user.target diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index c8bc2fd5bbc..c6f24fb14d9 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -111,6 +111,18 @@ matrix_synapse_rc_login: per_second: 0.17 burst_count: 3 +matrix_synapse_rc_admin_redaction: + per_second: 1 + burst_count: 50 + +matrix_synapse_rc_joins: + local: + per_second: 0.1 + burst_count: 3 + remote: + per_second: 0.01 + burst_count: 3 + matrix_synapse_rc_federation: window_size: 1000 sleep_limit: 10 @@ -260,6 +272,42 @@ matrix_synapse_metrics_port: 9100 # See https://github.com/matrix-org/synapse/blob/master/docs/manhole.md matrix_synapse_manhole_enabled: false +# Enable support for Synapse workers +matrix_synapse_workers_enabled: false + +# List of workers to spawn +matrix_synapse_workers_enabled_list: + - { worker: generic_worker, port: 18101 } + - { worker: generic_worker, port: 18102 } + - { worker: generic_worker, port: 18103 } + - { worker: generic_worker, port: 18104 } + - { worker: generic_worker, port: 18105 } + - { worker: generic_worker, port: 18106 } + - { worker: pusher, port: 18201 } + - { worker: appservice, port: 18301 } + - { worker: federation_sender, port: 18401 } + - { worker: media_repository, port: 18501 } + - { worker: user_dir, port: 18601 } + - { worker: frontend_proxy, port: 18701 } + +# The list of available workers (2020-08-28) +matrix_synapse_workers_avail_list: + - generic_worker + - pusher + - appservice + - federation_sender + - media_repository + - user_dir + - frontend_proxy + +# Redis information +matrix_synapse_redis_enabled: false +matrix_synapse_redis_host: "" +matrix_synapse_redis_port: 6379 +matrix_synapse_redis_password: "" + +# Port used for communication between main synapse process and workers +matrix_synapse_replication_http_port: 9093 # Send ERROR logs to sentry.io for easier tracking # To set this up: go to sentry.io, create a python project, and set diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml index 1ae7ade6989..68d9f5f647f 100644 --- a/roles/matrix-synapse/tasks/setup_synapse.yml +++ b/roles/matrix-synapse/tasks/setup_synapse.yml @@ -18,6 +18,8 @@ - import_tasks: "{{ role_path }}/tasks/ext/setup.yml" +- import_tasks: "{{ role_path }}/tasks/workers/setup.yml" + - import_tasks: "{{ role_path }}/tasks/synapse/setup.yml" - import_tasks: "{{ role_path }}/tasks/goofys/setup.yml" diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/workers/setup.yml new file mode 100644 index 00000000000..4951ac2d7d6 --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup.yml @@ -0,0 +1,8 @@ +--- +# a negative when condition will not actually prevent ansible from executing loops in imported tasks! + +- import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" + when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool" + +- import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml" + when: "not matrix_synapse_workers_enabled|bool" diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml new file mode 100644 index 00000000000..0031c236c61 --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -0,0 +1,42 @@ +--- + +- name: Ensure synapse worker base service file installed + template: + src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" + mode: 0644 + register: matrix_synapse_worker_systemd_service_result + +- name: Ensure previous worker service symlinks are cleaned (FIXME) + file: + path: "{{ item.root + '/' + item.path }}" + state: absent + when: + - matrix_synapse_workers_enabled|bool + - item.state == 'link' + - item.path is match('matrix-synapse-worker@.*\\.service') + with_filetree: + - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" + +- name: Ensure systemd reloaded the worker service unit + service: + daemon_reload: yes + +- name: Ensure individual worker service symlinks exist + service: + name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" + enabled: true + with_items: "{{ matrix_synapse_workers_enabled_list }}" + +- name: Ensure creation of specific worker configs + template: + src: "{{ role_path }}/templates/synapse/worker.yaml.j2" + dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml" + with_list: "{{ matrix_synapse_workers_enabled_list }}" + +- name: Add workers to synapse.wants list + set_fact: + matrix_synapse_systemd_wanted_services_list: > + {{ matrix_synapse_systemd_wanted_services_list + + ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }} + with_items: "{{ matrix_synapse_workers_enabled_list }}" diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml new file mode 100644 index 00000000000..d1e7e3b5610 --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -0,0 +1,38 @@ +--- + +- name: Populate service facts + service_facts: + +- name: Ensure any worker services are stopped + service: + name: "{{ item.key }}" + state: stopped + with_dict: "{{ ansible_facts.services|default({})|dict2items|selectattr('key', 'match', 'matrix-synapse-worker@.+\\.service')|list|items2dict }}" + +# As we cannot know the ports of workers removed from the enabled_list.. +# => .. just kill them all (FIXME?) +- name: Ensure previous worker service symlinks are cleaned + file: + path: "{{ item.root + '/' + item.path }}" + state: absent + when: + - not matrix_synapse_workers_enabled|bool + - item.state == 'link' + - item.path is match('matrix-synapse-worker@.*\\.service') + with_filetree: + - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" + +- name: Ensure synapse worker base service file gets removed + file: + path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" + state: absent + register: matrix_synapse_worker_systemd_service_result + +- name: Remove workers from synapse.wants list + set_fact: + matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}" + with_items: "{{ matrix_synapse_workers_avail_list }}" + +- name: Ensure systemd noticed removal of worker service units + service: + daemon_reload: yes diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index d02d743e896..508da3794c1 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -223,6 +223,39 @@ listeners: type: manhole {% endif %} +{% if matrix_synapse_workers_enabled %} + # c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md + # HTTP replication: for the workers to send data to the main synapse process + - port: {{ matrix_synapse_replication_http_port }} + bind_addresses: ['0.0.0.0'] + type: http + resources: + - names: [replication] + +# c.f. https://github.com/matrix-org/synapse/tree/master/contrib/systemd-with-workers/README.md +worker_app: synapse.app.homeserver + +# thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/ +# reduce the main worker's offerings to core homeserver business +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list %} +notify_appservices: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list %} +send_federation: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %} +enable_media_repo: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list %} +start_pushers: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %} +update_user_directory: false +{% endif %} + +# rather let systemd handle the forking +daemonize: false +{% endif %} # Forward extremities can build up in a room due to networking delays between # homeservers. Once this happens in a large room, calculation of the state of @@ -739,6 +772,7 @@ rc_login: {{ matrix_synapse_rc_login|to_json }} #rc_admin_redaction: # per_second: 1 # burst_count: 50 +rc_admin_redaction: {{ matrix_synapse_rc_admin_redaction|to_json }} # #rc_joins: # local: @@ -747,7 +781,7 @@ rc_login: {{ matrix_synapse_rc_login|to_json }} # remote: # per_second: 0.01 # burst_count: 3 - +rc_joins: {{ matrix_synapse_rc_joins|to_json }} # Ratelimiting settings for incoming federation # @@ -2426,16 +2460,16 @@ opentracing: redis: # Uncomment the below to enable Redis support. # - #enabled: true + enabled: {{ matrix_synapse_redis_enabled }} # Optional host and port to use to connect to redis. Defaults to # localhost and 6379 # - #host: localhost - #port: 6379 + host: {{ matrix_synapse_redis_host }} + port: {{ matrix_synapse_redis_port }} # Optional password if configured on the Redis instance # - #password: + password: {{ matrix_synapse_redis_password }} # vim:ft=yaml diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 new file mode 100644 index 00000000000..a46517c4e83 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 @@ -0,0 +1,29 @@ +#jinja2: lstrip_blocks: "True" +# c.f. https://github.com/matrix-org/synapse/pull/4662 +[Unit] +Description=Synapse Matrix Worker +After=matrix-synapse.service +BindsTo=matrix-synapse.service + +[Service] +Type=simple + +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre=/bin/sleep 5 + +# systemd ftw 🤦‍♂️ +# https://github.com/systemd/systemd/issues/14895#issuecomment-594123923 +ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \ + exec /usr/bin/docker exec \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + matrix-synapse \ + python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml" + +ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i +KillMode=process +Restart=always +RestartSec=10 +SyslogIdentifier=matrix-synapse-%i + +[Install] +WantedBy=matrix-synapse.service diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 index 47786eeee17..9a5ce8e448a 100644 --- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 @@ -43,6 +43,11 @@ ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-synapse \ {% if matrix_synapse_manhole_enabled and matrix_synapse_container_manhole_api_host_bind_port %} -p {{ matrix_synapse_container_manhole_api_host_bind_port }}:9000 \ {% endif %} + {% if matrix_synapse_workers_enabled %} + {% for worker in matrix_synapse_workers_enabled_list %} + -p {{ worker.port }}:{{ worker.port }} \ + {% endfor %} + {% endif %} -v {{ matrix_synapse_config_dir_path }}:/data:ro \ -v {{ matrix_synapse_storage_path }}:/matrix-media-store-parent:slave \ {% for volume in matrix_synapse_container_additional_volumes %} diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 new file mode 100644 index 00000000000..c99e97cdcb4 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -0,0 +1,27 @@ +#jinja2: lstrip_blocks: "True" +worker_app: synapse.app.{{ item.worker }} + +worker_replication_host: 127.0.0.1 +worker_replication_http_port: {{ matrix_synapse_replication_http_port }} + +{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %} +worker_listeners: + - type: http + port: {{ item.port }} + resources: + - names: +{% if item.worker in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %} + - client + - federation +{% elif item.worker in [ 'media_repository' ] %} + - media +{% endif %} +{% endif %} + +{% if item.worker == 'frontend_proxy' %} +worker_main_http_uri: http://127.0.0.1:8008 +{% endif %} + +worker_daemonize: false +worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid +worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config diff --git a/setup.yml b/setup.yml index 68c4e4eee73..c51d41889d7 100755 --- a/setup.yml +++ b/setup.yml @@ -7,6 +7,7 @@ - matrix-base - matrix-mailer - matrix-postgres + - matrix-redis - matrix-corporal - matrix-bridge-appservice-discord - matrix-bridge-appservice-slack