diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index be60283..186151f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -71,7 +71,15 @@ jobs: sudo add-apt-repository ppa:mozillateam/ppa printf 'Package: *\nPin: release o=LP-PPA-mozillateam\nPin-Priority: 1001' | sudo tee /etc/apt/preferences.d/mozilla-firefox - sudo apt install -y wget nodejs unzip firefox nginx + sudo apt install -y wget nodejs unzip firefox + + # if nginx is already installed, remove it + sudo apt remove -y nginx nginx-common nginx-core + + sudo add-apt-repository ppa:ondrej/nginx-mainline -y + sudo apt update -y + sudo apt install -y nginx libnginx-mod-brotli + pip install --upgrade pip pip install wheel diff --git a/doc/setup.md b/doc/setup.md index 1c72c4a..6de7556 100644 --- a/doc/setup.md +++ b/doc/setup.md @@ -13,7 +13,8 @@ Clone that application, and then proceed with the following instructions. ### On macOS -- Using [Homebrew](http://brew.sh/): `brew install supervisor nginx postgresql node` +- Using [Homebrew](http://brew.sh/): `brew install supervisor postgresql node` + - If you want to use [brotli compression](https://en.wikipedia.org/wiki/Brotli) with NGINX (better compression rates for the frontend), you can install NGINX with the `ngx_brotli` module with this command: `brew tap denji/nginx && brew install nginx-full --with-brotli`. _If you already had NGINX installed, you may need to uninstall it first with `brew unlink nginx`._ Otherwise, you can install NGINX normally with `brew install nginx`. - Start the postgresql server: - to start automatically at login: `brew services start postgresql` - to start manually: `pg_ctl -D /usr/local/var/postgres start` @@ -37,7 +38,19 @@ See [below](#configuration) for more information on modifying the baselayer conf ### On Linux - Using `apt-get`: - `sudo apt-get install nginx supervisor postgresql libpq-dev npm nodejs-legacy` + `sudo apt-get install supervisor postgresql libpq-dev npm nodejs-legacy` + + If you want to use [brotli compression](https://en.wikipedia.org/wiki/Brotli) with NGINX (better compression rates for the frontend), you have to install NGINX and the brotli module from another source with: + + ``` + sudo apt remove -y nginx nginx-common nginx-core + sudo add-apt-repository ppa:ondrej/nginx-mainline -y + sudo apt update -y + sudo apt install -y nginx libnginx-mod-brotli + ``` + + Otherwise, you can install NGINX normally with `sudo apt-get install nginx`. + - It may be necessary to configure your database permissions: at the end of your `pg_hba.conf` (typically in `/etc/postgresql/13.3/main` or `/var/lib/pgsql/data`), add the following lines and restart PostgreSQL diff --git a/services/nginx/mime.types b/services/nginx/mime.types index 4a6286f..25d8e89 100644 --- a/services/nginx/mime.types +++ b/services/nginx/mime.types @@ -57,6 +57,7 @@ application/geopackage+sqlite3 gpkg; application/gltf-buffer glbin glbuf; application/gml+xml gml; application/gzip gz tgz; +application/br br; application/hyperstudio stk; application/inkml+xml ink inkml; application/ipfix ipfix; diff --git a/services/nginx/nginx.conf.template b/services/nginx/nginx.conf.template index e70577d..bc8846a 100644 --- a/services/nginx/nginx.conf.template +++ b/services/nginx/nginx.conf.template @@ -1,6 +1,12 @@ error_log log/error.log error; pid run/nginx.pid; +{% if fill_config_feature.nginx_brotli.dynamic and fill_config_feature.nginx_brotli.modules_path %} +load_module {{ fill_config_feature.nginx_brotli.modules_path }}/ngx_http_brotli_filter_module.so; # for compressing responses on-the-fly +load_module {{ fill_config_feature.nginx_brotli.modules_path }}/ngx_http_brotli_static_module.so; # for serving pre-compressed files +{% endif %} + + # Choose number of NGINX worker processes based on number of CPUs worker_processes auto; @@ -21,6 +27,19 @@ http { text/javascript application/javascript; + {% if fill_config_feature.nginx_brotli.installed %} + # also enable brotli compression + brotli on; + brotli_comp_level 6; + brotli_types text/plain + text/css + application/json + application/x-javascript + application/xml + text/javascript + application/javascript; + {% endif %} + # Only retry if there was a communication error, not a timeout # on the Tornado server (to avoid propagating "queries of death" # to all frontends) diff --git a/tools/fill_conf_values.py b/tools/fill_conf_values.py index 6a072d7..65cb6ef 100755 --- a/tools/fill_conf_values.py +++ b/tools/fill_conf_values.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os +import subprocess import jinja2 from status import status @@ -36,12 +37,89 @@ def hash_filter(string, htype): return h.hexdigest() +def nginx_brotli_installed(): + """Check if the nginx brotli module is installed + + Returns + ------- + installed : bool + True if the nginx brotli module is installed, False otherwise + dynamic : bool + True if the module is dynamically loaded, False otherwise + modules_path : str + The directory where the nginx modules are located if dynamic loading is used + """ + + installed = False + dynamic = False + modules_path = None + + try: + output = subprocess.check_output( + ["nginx", "-V"], stderr=subprocess.STDOUT + ).decode("utf-8") + # Option 1: installed at compilation: always loaded + if ( + "--add-module" in output + and "brotli" in output.split("--add-module")[1].strip() + ): + installed = True + # Option 2: installed dynamically at compilation or later: has to be loaded + else: + # a. find the modules path + config_path = ( + str(output.split("--conf-path=")[1].split(" ")[0]).strip() + if "--conf-path" in output + else None + ) + modules_path = ( + str(output.split("--modules-path=")[1].split(" ")[0]).strip() + if "--modules-path" in output + else None + ) + # if there's no modules path, try to guess it from the config path + if config_path and not modules_path: + modules_path = os.path.dirname(config_path).replace( + "nginx.conf", "modules" + ) + if not modules_path or not os.path.isdir(modules_path): + modules_path = None + + # b. check if there is a brotli module in the modules path + if modules_path: + modules_path = modules_path.rstrip("/") + if all( + os.path.isfile(os.path.join(modules_path, f)) + for f in [ + "ngx_http_brotli_filter_module.so", + "ngx_http_brotli_static_module.so", + ] + ): + installed = True + dynamic = True + else: + installed = False + dynamic = False + modules_path = None + except subprocess.CalledProcessError: + pass + return installed, dynamic, modules_path + + custom_filters = {"md5sum": md5sum, "version": version, "hash": hash_filter} def fill_config_file_values(template_paths): log("Compiling configuration templates") env, cfg = load_env() + installed, dynamic, modules_path = nginx_brotli_installed() + cfg["fill_config_feature"] = { + "nginx_brotli": { + "installed": installed, + "dynamic": dynamic, + "modules_path": modules_path, + } + } for template_path in template_paths: with status(template_path):