Support ACME for certificate provisioning

changelog.d/4384.feature

@@ -0,0 +1 @@
+Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt).

scripts-dev/build_debian_packages

@@ -10,12 +10,12 @@
 # can be passed on the commandline for debugging.
 
 import argparse
-from concurrent.futures import ThreadPoolExecutor
 import os
 import signal
 import subprocess
 import sys
 import threading
+from concurrent.futures import ThreadPoolExecutor
 
 DISTS = (
     "debian:stretch",

synapse/app/homeserver.py

@@ -13,6 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import gc
 import logging
 import os
@@ -324,8 +325,13 @@ def setup(config_options):
 
     events.USE_FROZEN_DICTS = config.use_frozen_dicts
 
-    tls_server_context_factory = context_factory.ServerContextFactory(config)
-    tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
+    # These will be loaded in later, once we have provisioned keys via ACME.
+    if config.acme_enabled:
+        tls_server_context_factory = None
+        tls_client_options_factory = None
+    else:
+        tls_server_context_factory = context_factory.ServerContextFactory(config)
+        tls_client_options_factory = context_factory.ClientTLSOptionsFactory(config)
 
     database_engine = create_engine(config.database_config)
     config.database_config["args"]["cp_openfun"] = database_engine.on_new_connection
@@ -361,9 +367,37 @@ def setup(config_options):
     logger.info("Database prepared in %s.", config.database_config['name'])
 
     hs.setup()
-    hs.start_listening()
+    # If configured, start up the ACME listener. We will need to check if we
+    # have a certificate, first.
+    if hs.config.acme_enabled:
+        acme = hs.get_acme_handler()
+        acme.start_listening()
+    else:
+        hs.start_listening()
 
     def start():
+        if hs.config.acme_enabled:
+            is_valid_cert = acme.is_disk_cert_valid()
+            if not is_valid_cert:
+                d = acme._issuer._ensure_registered()
+                d.addCallback(
+                    lambda _: acme.provision_certificate(hs.config.server_name)
+                )
+            else:
+                d = defer.succeed(True)
+
+            def _load_context_factories(_):
+                hs.tls_server_context_factory = context_factory.ServerContextFactory(
+                    config
+                )
+                hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
+                    config
+                )
+
+            d.addCallback(lambda _: hs.config._read_certificate())
+            d.addCallback(_load_context_factories)
+            d.addCallback(lambda _: hs.start_listening())
+
         hs.get_pusherpool().start()
         hs.get_datastore().start_profiling()
         hs.get_datastore().start_doing_background_updates()

synapse/config/_base.py

@@ -367,7 +367,7 @@ def read_config_files(self, config_files, keys_directory=None, generate_keys=Fal
         if not keys_directory:
             keys_directory = os.path.dirname(config_files[-1])
 
-        config_dir_path = os.path.abspath(keys_directory)
+        self.config_dir_path = os.path.abspath(keys_directory)
 
         specified_config = {}
         for config_file in config_files:
@@ -379,7 +379,7 @@ def read_config_files(self, config_files, keys_directory=None, generate_keys=Fal
 
         server_name = specified_config["server_name"]
         config_string = self.generate_config(
-            config_dir_path=config_dir_path,
+            config_dir_path=self.config_dir_path,
             data_dir_path=os.getcwd(),
             server_name=server_name,
             generate_secrets=False,

synapse/config/tls.py

@@ -28,25 +28,57 @@
 
 class TlsConfig(Config):
     def read_config(self, config):
-        self.tls_certificate = self.read_tls_certificate(
-            config.get("tls_certificate_path")
+
+        acme_config = config.get("acme", {})
+        self.acme_enabled = acme_config.get("enabled", False)
+        self.acme_url = acme_config.get(
+            "url", "https://acme-staging.api.letsencrypt.org/directory"
         )
-        self.tls_certificate_file = config.get("tls_certificate_path")
+        self.acme_client_key = config.get("client_key", self.config_dir_path)
+        self.acme_port = acme_config.get("port", 8449)
+        self.acme_host = acme_config.get("host", "127.0.0.1")
 
+        self.tls_certificate_file = config.get("tls_certificate_path")
+        self.tls_private_key_file = config.get("tls_private_key_path")
+        self.tls_dh_params_file = config.get("tls_dh_params_path")
+        self._original_tls_fingerprints = config["tls_fingerprints"]
+        self.tls_fingerprints = list(self._original_tls_fingerprints)
         self.no_tls = config.get("no_tls", False)
 
-        if self.no_tls:
-            self.tls_private_key = None
-        else:
-            self.tls_private_key = self.read_tls_private_key(
-                config.get("tls_private_key_path")
-            )
-
         self.tls_dh_params_path = self.check_file(
-            config.get("tls_dh_params_path"), "tls_dh_params"
+            self.tls_dh_params_file, "tls_dh_params"
         )
 
-        self.tls_fingerprints = config["tls_fingerprints"]
+        # This config option applies to non-federation HTTP clients
+        # (e.g. for talking to recaptcha, identity servers, and such)
+        # It should never be used in production, and is intended for
+        # use only when running tests.
+        self.use_insecure_ssl_client_just_for_testing_do_not_use = config.get(
+            "use_insecure_ssl_client_just_for_testing_do_not_use"
+        )
+
+        self.tls_certificate = None
+        self.tls_private_key = None
+
+        # If we are using ACME, do not read the certificate yet. That will be
+        # done later, and will trigger the reading code.
+        if not self.acme_enabled:
+            self._read_certificate()
+
+    def _read_certificate(self):
+        """
+        Read the certificates from disk.
+        """
+        self.tls_certificate = self.read_tls_certificate(
+            self.tls_certificate_file
+        )
+
+        if not self.no_tls:
+            self.tls_private_key = self.read_tls_private_key(
+                self.tls_private_key_file
+            )
+
+        self.tls_fingerprints = list(self._original_tls_fingerprints)
 
         # Check that our own certificate is included in the list of fingerprints
         # and include it if it is not.
@@ -59,14 +91,6 @@ def read_config(self, config):
         if sha256_fingerprint not in sha256_fingerprints:
             self.tls_fingerprints.append({u"sha256": sha256_fingerprint})
 
-        # This config option applies to non-federation HTTP clients
-        # (e.g. for talking to recaptcha, identity servers, and such)
-        # It should never be used in production, and is intended for
-        # use only when running tests.
-        self.use_insecure_ssl_client_just_for_testing_do_not_use = config.get(
-            "use_insecure_ssl_client_just_for_testing_do_not_use"
-        )
-
     def default_config(self, config_dir_path, server_name, **kwargs):
         base_key_name = os.path.join(config_dir_path, server_name)
 
@@ -118,6 +142,17 @@ def default_config(self, config_dir_path, server_name, **kwargs):
         #
         tls_fingerprints: []
         # tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
+
+        ## Support for ACME certificate auto-provisioning.
+        # acme:
+        #    enabled: false
+        ##   ACME path. Default: https://acme-staging.api.letsencrypt.org/directory
+        #    url: 'https://acme-v01.api.letsencrypt.org/directory'
+        ##   Port number (to listen for the HTTP-01 challenge).
+        ##   Using port 80 requires utilising something like authbind, or proxying to it.
+        #    port: 8449
+        ##   Hosts to bind to, comma separated.
+        #    host: '127.0.0.1'
         """ % locals()
 
     def read_tls_certificate(self, cert_path):

synapse/handlers/_base.py

@@ -46,6 +46,7 @@ def __init__(self, hs):
         self.distributor = hs.get_distributor()
         self.ratelimiter = hs.get_ratelimiter()
         self.clock = hs.get_clock()
+        self.reactor = hs.get_reactor()
         self.hs = hs
 
         self.server_name = hs.hostname

synapse/handlers/acme.py

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 New Vector Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+import attr
+from zope.interface import implementer
+
+from OpenSSL import crypto
+from twisted.internet import defer
+from twisted.internet.endpoints import serverFromString
+from twisted.python.filepath import FilePath
+from twisted.python.url import URL
+from twisted.web import server, static
+from twisted.web.resource import Resource
+
+from ._base import BaseHandler
+
+logger = logging.getLogger(__name__)
+
+try:
+    from txacme.interfaces import ICertificateStore
+
+    @attr.s
+    @implementer(ICertificateStore)
+    class ErsatzStore(object):
+        """
+        A store that only stores in memory.
+        """
+
+        certs = attr.ib(default=attr.Factory(dict))
+
+        def store(self, server_name, pem_objects):
+            self.certs[server_name] = b''.join(o.as_bytes() for o in pem_objects)
+            return defer.succeed(None)
+
+
+except ImportError:
+    # txacme is missing
+    pass
+
+
+class AcmeHandler(BaseHandler):
+    def __init__(self, hs):
+        super(AcmeHandler, self).__init__(hs)
+
+    def is_disk_cert_valid(self):
+        """
+        Is the certificate we have on disk valid?
+        """
+        try:
+            tls_certificate = self.hs.config.read_tls_certificate(
+                self.hs.config.tls_certificate_file
+            )
+        except Exception:
+            logger.warning("Certificate does not exist, will reprovision....")
+            return False
+
+        expired = tls_certificate.has_expired()
+
+        if expired:
+            logger.warning("Certificate is expired, will reprovision...")
+            return False
+
+        return True
+
+    def start_listening(self):
+
+        # Configure logging for txacme, if you need to debug
+        # from eliot import add_destinations
+        # from eliot.twisted import TwistedDestination
+        #
+        # add_destinations(TwistedDestination())
+
+        from txacme.challenges import HTTP01Responder
+        from txacme.service import AcmeIssuingService
+        from txacme.endpoint import load_or_create_client_key
+        from txacme.client import Client
+        from josepy.jwa import RS256
+
+        self._store = ErsatzStore()
+        responder = HTTP01Responder()
+
+        self._issuer = AcmeIssuingService(
+            cert_store=self._store,
+            client_creator=(
+                lambda: Client.from_url(
+                    reactor=self.reactor,
+                    url=URL.from_text(self.hs.config.acme_url),
+                    key=load_or_create_client_key(
+                        FilePath(self.hs.config.acme_client_key)
+                    ),
+                    alg=RS256,
+                )
+            ),
+            clock=self.reactor,
+            responders=[responder],
+        )
+
+        well_known = Resource()
+        well_known.putChild(b'acme-challenge', responder.resource)
+        responder_resource = Resource()
+        responder_resource.putChild(b'.well-known', well_known)
+        responder_resource.putChild(b'check', static.Data(b'OK', b'text/plain'))
+
+        srv = server.Site(responder_resource)
+
+        for host in self.hs.config.acme_host.split(","):
+            logger.info(
+                "Listening for ACME requests on %s:%s", host, self.hs.config.acme_port
+            )
+            endpoint = serverFromString(
+                self.reactor, "tcp:%s:interface=%s" % (self.hs.config.acme_port, host)
+            )
+            endpoint.listen(srv)
+
+        self._issuer._registered = False
+
+    @defer.inlineCallbacks
+    def provision_certificate(self, hostname):
+
+        logger.warning("Reprovisioning %s", hostname)
+
+        try:
+            yield self._issuer.issue_cert(hostname)
+        except Exception:
+            logger.exception("Fail!")
+            raise
+        logger.warning("Reprovisioned %s, saving.", hostname)
+        cert_chain = self._store.certs[hostname]
+
+        try:
+            tls_private_key = crypto.load_privatekey(crypto.FILETYPE_PEM, cert_chain)
+            with open(self.hs.config.tls_private_key_file, "wb") as private_key_file:
+                private_key_pem = crypto.dump_privatekey(
+                    crypto.FILETYPE_PEM, tls_private_key
+                )
+                private_key_file.write(private_key_pem)
+
+            cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_chain)
+            with open(self.hs.config.tls_certificate_file, "wb") as certificate_file:
+                cert_pem = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
+                certificate_file.write(cert_pem)
+        except Exception:
+            logger.exception("Failed saving!")
+            raise
+
+        defer.returnValue(True)

synapse/python_dependencies.py

@@ -72,6 +72,7 @@
     # ConsentResource uses select_autoescape, which arrived in jinja 2.9
     "resources.consent": ["Jinja2>=2.9"],
 
+    "acme": ["txacme>=0.9.2"],
     "saml2": ["pysaml2>=4.5.0"],
     "url_preview": ["lxml>=3.5.0"],
     "test": ["mock>=2.0"],