Get matrix-corporal to play nicely with a Synapse worker setup

We do this by creating one more layer of indirection.

First we reach some generic vhost handling matrix.DOMAIN.
A bunch of override rules are added there (capturing traffic to send to
ma1sd, etc). nginx-status and similar generic things also live there.

We then proxy to the homeserver on some other vhost (only Synapse being
available right now, but repointing this to Dendrite or other will be
possible in the future).
Then that homeserver-specific vhost does its thing to proxy to the
homeserver. It may or may not use workers, etc.

Without matrix-corporal, the flow is now:
1. matrix.DOMAIN (matrix-nginx-proxy/matrix-domain.conf)
2. matrix-nginx-proxy/matrix-synapse.conf
3. matrix-synapse

With matrix-corporal enabled, it becomes:
1. matrix.DOMAIN (matrix-nginx-proxy/matrix-domain.conf)
2. matrix-corporal
3. matrix-nginx-proxy/matrix-synapse.conf
4. matrix-synapse

(matrix-corporal gets injected at step 2).

group_vars/matrix_servers

@@ -673,7 +673,8 @@ matrix_corporal_systemd_required_services_list: |
     (['matrix-synapse.service'])
   }}
 
-matrix_corporal_matrix_homeserver_api_endpoint: "http://matrix-synapse:8008"
+# This goes to Synapse's vhost
+matrix_corporal_matrix_homeserver_api_endpoint: "http://matrix-nginx-proxy:12080"
 
 matrix_corporal_matrix_auth_shared_secret: "{{ matrix_synapse_ext_password_provider_shared_secret_auth_shared_secret }}"
 
@@ -885,7 +886,7 @@ matrix_ma1sd_synapsesql_connection: //{{ matrix_synapse_database_host }}/{{ matr
 
 matrix_ma1sd_dns_overwrite_enabled: true
 matrix_ma1sd_dns_overwrite_homeserver_client_name: "{{ matrix_server_fqn_matrix }}"
-matrix_ma1sd_dns_overwrite_homeserver_client_value: "http://{{ 'matrix-corporal:41080' if matrix_corporal_enabled else 'matrix-synapse:8008' }}"
+matrix_ma1sd_dns_overwrite_homeserver_client_value: "http://{{ matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container }}"
 
 # By default, we send mail through the `matrix-mailer` service.
 matrix_ma1sd_threepid_medium_email_identity_from: "{{ matrix_mailer_sender_address }}"
@@ -932,8 +933,8 @@ matrix_ma1sd_database_password: "{{ matrix_synapse_macaroon_secret_key | passwor
 # If that's not the case, you may wish to disable this and take care of proxying yourself.
 matrix_nginx_proxy_enabled: true
 
-matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "{{ 'matrix-corporal:41080' if matrix_corporal_enabled else 'matrix-synapse:8008' }}"
-matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "{{ '127.0.0.1:41080' if matrix_corporal_enabled else '127.0.0.1:8008' }}"
+matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "{{ 'matrix-corporal:41080' if matrix_corporal_enabled else 'matrix-nginx-proxy:12080' }}"
+matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "{{ '127.0.0.1:41080' if matrix_corporal_enabled else '127.0.0.1:12080' }}"
 matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: "{{ matrix_synapse_max_upload_size_mb }}"
 
 matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled: "{{ matrix_synapse_admin_enabled }}"
@@ -956,8 +957,12 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:809
 # By default, we do TLS termination for the Matrix Federation API (port 8448) at matrix-nginx-proxy.
 # Unless this is handled there OR Synapse's federation listener port is disabled, we'll reverse-proxy.
 matrix_nginx_proxy_proxy_matrix_federation_api_enabled: "{{ matrix_synapse_federation_port_enabled and not matrix_synapse_tls_federation_listener_enabled }}"
-matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-synapse:8048"
-matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "127.0.0.1:8048"
+matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-nginx-proxy:12088"
+matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "127.0.0.1:12088"
+
+# Settings controlling matrix-synapse-proxy.conf
+matrix_nginx_proxy_proxy_synapse_enabled: "{{ matrix_synapse_enabled }}"
+matrix_nginx_proxy_proxy_synapse_federation_api_enabled: "{{ matrix_nginx_proxy_proxy_matrix_federation_api_enabled }}"
 
 matrix_nginx_proxy_container_federation_host_bind_port: "{{ matrix_federation_public_port }}"
 

roles/matrix-nginx-proxy/defaults/main.yml

@@ -99,6 +99,10 @@ matrix_nginx_proxy_access_log_enabled: true
 matrix_nginx_proxy_proxy_riot_compat_redirect_enabled: false
 matrix_nginx_proxy_proxy_riot_compat_redirect_hostname: "riot.{{ matrix_domain }}"
 
+# Controls whether proxying the Synapse domain should be done.
+matrix_nginx_proxy_proxy_synapse_enabled: false
+matrix_nginx_proxy_proxy_synapse_hostname: "matrix-nginx-proxy"
+
 # Controls whether proxying the Element domain should be done.
 matrix_nginx_proxy_proxy_element_enabled: false
 matrix_nginx_proxy_proxy_element_hostname: "{{ matrix_server_fqn_element }}"
@@ -146,8 +150,13 @@ matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key: ""
 
 # The addresses where the Matrix Client API is.
 # Certain extensions (like matrix-corporal) may override this in order to capture all traffic.
-matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "matrix-synapse:8008"
-matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "127.0.0.1:8008"
+matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "matrix-nginx-proxy:12080"
+matrix_nginx_proxy_proxy_matrix_client_api_addr_sans_container: "127.0.0.1:12080"
+
+# The addresses where the Matrix Client API is, when using Synapse.
+matrix_nginx_proxy_proxy_synapse_client_api_addr_with_container: "matrix-synapse:8008"
+matrix_nginx_proxy_proxy_synapse_client_api_addr_sans_container: "127.0.0.1:8008"
+
 # This needs to be equal or higher than the maximum upload size accepted by Synapse.
 matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb: 50
 
@@ -185,34 +194,41 @@ matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to_domain: ""
 
 # Controls whether proxying for the Matrix Federation API should be done.
 matrix_nginx_proxy_proxy_matrix_federation_api_enabled: false
-matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-synapse:8048"
-matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "localhost:8048"
+matrix_nginx_proxy_proxy_matrix_federation_api_addr_with_container: "matrix-nginx-proxy:12088"
+matrix_nginx_proxy_proxy_matrix_federation_api_addr_sans_container: "localhost:12088"
 matrix_nginx_proxy_proxy_matrix_federation_api_client_max_body_size_mb: "{{ (matrix_nginx_proxy_proxy_matrix_client_api_client_max_body_size_mb | int) * 3 }}"
 matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate: "{{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_matrix_hostname }}/fullchain.pem"
 matrix_nginx_proxy_proxy_matrix_federation_api_ssl_certificate_key: "{{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_proxy_matrix_hostname }}/privkey.pem"
 
+# The addresses where the Federation API is, when using Synapse.
+matrix_nginx_proxy_proxy_synapse_federation_api_addr_with_container: "matrix-synapse:8048"
+matrix_nginx_proxy_proxy_synapse_federation_api_addr_sans_container: "localhost:8048"
+
 # The tmpfs at /tmp needs to be large enough to handle multiple concurrent file uploads.
 matrix_nginx_proxy_tmp_directory_size_mb: "{{ (matrix_nginx_proxy_proxy_matrix_federation_api_client_max_body_size_mb | int) * 50 }}"
 
-# A list of strings containing additional configuration blocks to add to the nginx http's server configuration.
+# A list of strings containing additional configuration blocks to add to the nginx http's server configuration (nginx-http.conf).
 matrix_nginx_proxy_proxy_http_additional_server_configuration_blocks: []
 
-# A list of strings containing additional configuration blocks to add to the matrix synapse's server configuration.
+# A list of strings containing additional configuration blocks to add to the base matrix server configuration (matrix-domain.conf).
 matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks: []
 
-# A list of strings containing additional configuration blocks to add to Riot's server configuration.
+# A list of strings containing additional configuration blocks to add to the synapse's server configuration (matrix-synapse.conf).
+matrix_nginx_proxy_proxy_synapse_additional_server_configuration_blocks: []
+
+# A list of strings containing additional configuration blocks to add to Riot's server configuration (matrix-riot-web.conf).
 matrix_nginx_proxy_proxy_riot_additional_server_configuration_blocks: []
 
-# A list of strings containing additional configuration blocks to add to Element's server configuration.
+# A list of strings containing additional configuration blocks to add to Element's server configuration (matrix-client-element.conf).
 matrix_nginx_proxy_proxy_element_additional_server_configuration_blocks: []
 
-# A list of strings containing additional configuration blocks to add to Dimension's server configuration.
+# A list of strings containing additional configuration blocks to add to Dimension's server configuration (matrix-dimension.conf).
 matrix_nginx_proxy_proxy_dimension_additional_server_configuration_blocks: []
 
-# A list of strings containing additional configuration blocks to add to Jitsi's server configuration.
+# A list of strings containing additional configuration blocks to add to Jitsi's server configuration (matrix-jitsi.conf).
 matrix_nginx_proxy_proxy_jitsi_additional_server_configuration_blocks: []
 
-# A list of strings containing additional configuration blocks to add to the base domain server configuration.
+# A list of strings containing additional configuration blocks to add to the base domain server configuration (matrix-base-domain.conf).
 matrix_nginx_proxy_proxy_domain_additional_server_configuration_blocks: []
 
 # Specifies the SSL configuration that should be used for the SSL protocols and ciphers

roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml

@@ -45,12 +45,18 @@
     mode: 0644
   when: matrix_nginx_proxy_enabled|bool
 
-- name: Ensure Matrix nginx-proxy configuration for matrix domain exists
+- name: Ensure Matrix nginx-proxy configuration for matrix-synapse exists
   template:
     src: "{{ role_path }}/templates/nginx/conf.d/matrix-synapse.conf.j2"
     dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-synapse.conf"
     mode: 0644
-  when: matrix_nginx_proxy_proxy_matrix_enabled|bool
+  when: matrix_nginx_proxy_proxy_synapse_enabled|bool
+
+- name: Ensure Matrix nginx-proxy configuration for matrix-synapse deleted
+  file:
+    path: "{{ matrix_nginx_proxy_confd_path }}/matrix-synapse.conf"
+    state: absent
+  when: "not matrix_nginx_proxy_proxy_synapse_enabled|bool"
 
 - name: Ensure Matrix nginx-proxy configuration for Element domain exists
   template:
@@ -80,6 +86,12 @@
     mode: 0644
   when: matrix_nginx_proxy_proxy_jitsi_enabled|bool
 
+- name: Ensure Matrix nginx-proxy configuration for Matrix domain exists
+  template:
+    src: "{{ role_path }}/templates/nginx/conf.d/matrix-domain.conf.j2"
+    dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-domain.conf"
+    mode: 0644
+
 - name: Ensure Matrix nginx-proxy data directory for base domain exists
   file:
     path: "{{ matrix_nginx_proxy_data_path }}/matrix-domain"
@@ -100,8 +112,8 @@
 
 - name: Ensure Matrix nginx-proxy configuration for base domain exists
   template:
-    src: "{{ role_path }}/templates/nginx/conf.d/matrix-domain.conf.j2"
-    dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-domain.conf"
+    src: "{{ role_path }}/templates/nginx/conf.d/matrix-base-domain.conf.j2"
+    dest: "{{ matrix_nginx_proxy_confd_path }}/matrix-base-domain.conf"
     mode: 0644
   when: matrix_nginx_proxy_base_domain_serving_enabled|bool
 
@@ -161,7 +173,7 @@
 
 - name: Ensure Matrix nginx-proxy configuration for matrix domain deleted
   file:
-    path: "{{ matrix_nginx_proxy_confd_path }}/matrix-synapse.conf"
+    path: "{{ matrix_nginx_proxy_confd_path }}/matrix-domain.conf"
     state: absent
   when: "not matrix_nginx_proxy_proxy_matrix_enabled|bool"
 
@@ -191,7 +203,7 @@
 
 - name: Ensure Matrix nginx-proxy configuration for base domain deleted
   file:
-    path: "{{ matrix_nginx_proxy_confd_path }}/matrix-domain.conf"
+    path: "{{ matrix_nginx_proxy_confd_path }}/matrix-base-domain.conf"
     state: absent
   when: "not matrix_nginx_proxy_base_domain_serving_enabled|bool"
 

roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-base-domain.conf.j2

@@ -0,0 +1,70 @@
+#jinja2: lstrip_blocks: "True"
+
+{% macro render_vhost_directives() %}
+	root /nginx-data/matrix-domain;
+
+	gzip on;
+	gzip_types text/plain application/json;
+	{% for configuration_block in matrix_nginx_proxy_proxy_domain_additional_server_configuration_blocks %}
+		{{- configuration_block }}
+	{% endfor %}
+
+	location /.well-known/matrix {
+		root {{ matrix_static_files_base_path }};
+		{#
+			A somewhat long expires value is used to prevent outages
+			in case this is unreachable due to network failure.
+		#}
+		expires 4h;
+		default_type application/json;
+		add_header Access-Control-Allow-Origin *;
+	}
+{% endmacro %}
+
+server {
+	listen {{ 8080 if matrix_nginx_proxy_enabled else 80 }};
+
+	server_name {{ matrix_nginx_proxy_base_domain_hostname }};
+	server_tokens off;
+
+	{% if matrix_nginx_proxy_https_enabled %}
+		location /.well-known/acme-challenge {
+			{% if matrix_nginx_proxy_enabled %}
+				{# Use the embedded DNS resolver in Docker containers to discover the service #}
+				resolver 127.0.0.11 valid=5s;
+				set $backend "matrix-certbot:8080";
+				proxy_pass http://$backend;
+			{% else %}
+				{# Generic configuration for use outside of our container setup #}
+				proxy_pass http://127.0.0.1:{{ matrix_ssl_lets_encrypt_certbot_standalone_http_port }};
+			{% endif %}
+		}
+
+		location / {
+			return 301 https://$http_host$request_uri;
+		}
+	{% else %}
+		{{ render_vhost_directives() }}
+	{% endif %}
+}
+
+{% if matrix_nginx_proxy_https_enabled %}
+server {
+	listen {{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2;
+	listen [::]:{{ 8443 if matrix_nginx_proxy_enabled else 443 }} ssl http2;
+
+	server_name {{ matrix_nginx_proxy_base_domain_hostname }};
+	server_tokens off;
+
+	ssl_certificate {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_base_domain_hostname }}/fullchain.pem;
+	ssl_certificate_key {{ matrix_ssl_config_dir_path }}/live/{{ matrix_nginx_proxy_base_domain_hostname }}/privkey.pem;
+
+	ssl_protocols {{ matrix_nginx_proxy_ssl_protocols }};
+	{% if matrix_nginx_proxy_ssl_ciphers != '' %}
+	ssl_ciphers {{ matrix_nginx_proxy_ssl_ciphers }};
+	{% endif %}
+	ssl_prefer_server_ciphers {{ matrix_nginx_proxy_ssl_prefer_server_ciphers }};
+
+	{{ render_vhost_directives() }}
+}
+{% endif %}