Brotli compression support

.github/workflows/test.yaml

@@ -45,7 +45,10 @@ jobs:
 
       - name: Install template app
         run: |
-          git clone https://github.com/cesium-ml/baselayer_template_app
+          git clone https://github.com/Theodlz/baselayer_template_app
+          cd baselayer_template_app
+          git checkout brotli
+          cd ..
           cp -rf baselayer baselayer_template_app/
 
       - uses: actions/cache@v3
@@ -71,7 +74,25 @@ jobs:
           sudo add-apt-repository ppa:mozillateam/ppa
           printf 'Package: *\nPin: release o=LP-PPA-mozillateam\nPin-Priority: 1001' | sudo tee /etc/apt/preferences.d/mozilla-firefox
 
-          sudo apt install -y wget nodejs unzip firefox nginx
+          sudo apt install -y wget nodejs unzip firefox
+
+          # if nginx is already installed, remove it
+          sudo apt remove -y nginx nginx-common nginx-core
+
+          # install nginx from source so we can add the brotli module
+          git clone --recursive https://github.com/google/ngx_brotli.git
+          wget https://nginx.org/download/nginx-1.24.0.tar.gz
+          tar zxf nginx-1.24.0.tar.gz
+          cd ngx_brotli/deps/brotli
+          mkdir out && cd out
+          cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_FLAGS="-Ofast -m64 -march=native -mtune=native -flto -funroll-loops -ffunction-sections -fdata-sections -Wl,--gc-sections" -DCMAKE_CXX_FLAGS="-Ofast -m64 -march=native -mtune=native -flto -funroll-loops -ffunction-sections -fdata-sections -Wl,--gc-sections" -DCMAKE_INSTALL_PREFIX=./installed ..
+          cmake --build . --config Release --target brotlienc
+          cd ../../../..
+          export CURRENT_DIR=$(pwd)
+          cd nginx-1.24.0
+          ./configure --sbin-path=/usr/bin/nginx --conf-path=/usr/local/nginx/nginx.conf --pid-path=/usr/local/nginx/nginx.pid --with-http_ssl_module --with-stream --with-mail=dynamic --with-http_realip_module --with-compat --add-module=${CURRENT_DIR}/ngx_brotli
+          sudo make && sudo make install
+          sudo nginx
 
           pip install --upgrade pip
           pip install wheel

doc/setup.md

@@ -13,7 +13,8 @@ Clone that application, and then proceed with the following instructions.
 
 ### On macOS
 
-- Using [Homebrew](http://brew.sh/): `brew install supervisor nginx postgresql node`
+- Using [Homebrew](http://brew.sh/): `brew install supervisor postgresql node`
+  - If you want to use [brotli compression](https://en.wikipedia.org/wiki/Brotli) with NGINX (better compression rates for the frontend), you can install NGINX with the `ngx_brotli` module with this command: `brew tap denji/nginx && brew install nginx-full --with-brotli`. *If you already had NGINX installed, you may need to uninstall it first with `brew unlink nginx`.*Otherwise, you can install NGINX normally with `brew install nginx`.
   - Start the postgresql server:
     - to start automatically at login: `brew services start postgresql`
     - to start manually: `pg_ctl -D /usr/local/var/postgres start`
@@ -37,7 +38,50 @@ See [below](#configuration) for more information on modifying the baselayer conf
 ### On Linux
 
 - Using `apt-get`:
-  `sudo apt-get install nginx supervisor postgresql libpq-dev npm nodejs-legacy`
+  `sudo apt-get install supervisor postgresql libpq-dev npm nodejs-legacy`
+
+  - If you want to run NGINX as is (without brotli), you can simply install it with `sudo apt-get install nginx`.
+    If want to use [brotli compression](https://en.wikipedia.org/wiki/Brotli) with NGINX (to have better compression rates for the frontend), you can install NGINX with the `ngx_brotli` module with these commands:
+
+  ```
+  # install nginx from source so we can add the brotli module
+  git clone --recursive https://github.com/google/ngx_brotli.git
+  wget https://nginx.org/download/nginx-1.24.0.tar.gz
+  tar zxf nginx-1.24.0.tar.gz
+  cd ngx_brotli/deps/brotli
+  mkdir out && cd out
+  cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_FLAGS="-Ofast -m64 -march=native -mtune=native -flto -funroll-loops -ffunction-sections -fdata-sections -Wl,--gc-sections" -DCMAKE_CXX_FLAGS="-Ofast -m64 -march=native -mtune=native -flto -funroll-loops -ffunction-sections -fdata-sections -Wl,--gc-sections" -DCMAKE_INSTALL_PREFIX=./installed ..
+  cmake --build . --config Release --target brotlienc
+  cd ../../../..
+  export CURRENT_DIR=$(pwd)
+  cd nginx-1.24.0
+  ./configure --sbin-path=/usr/sbin/nginx --conf-path=/usr/local/nginx/nginx.conf --pid-path=/usr/local/nginx/nginx.pid --with-http_ssl_module --with-stream --with-mail=dynamic --with-http_realip_module --with-compat --add-module=${CURRENT_DIR}/ngx_brotli
+  sudo make && sudo make install
+  ```
+
+  To run it as a service, create an Nginx systemd unit file by running `sudo nano /lib/systemd/system/nginx.service` and adding the following content:
+
+  ```
+  [Unit]
+  Description=The NGINX HTTP and reverse proxy server
+  After=syslog.target network-online.target remote-fs.target nss-lookup.target
+  Wants=network-online.target
+
+  [Service]
+  Type=forking
+  PIDFile=/usr/local/nginx/nginx.pid
+  ExecStartPre=/usr/sbin/nginx -t
+  ExecStart=/usr/sbin/nginx
+  ExecReload=/usr/sbin/nginx -s reload
+  ExecStop=/bin/kill -s QUIT $MAINPID
+  PrivateTmp=true
+
+  [Install]
+  WantedBy=multi-user.target
+  ```
+
+  Then run `sudo systemctl start nginx` to start the server. To start it automatically at boot, run `sudo systemctl enable nginx`.
+
 - It may be necessary to configure your database permissions: at
   the end of your `pg_hba.conf` (typically in `/etc/postgresql/13.3/main` or `/var/lib/pgsql/data`),
   add the following lines and restart PostgreSQL

services/nginx/mime.types

@@ -57,6 +57,7 @@ application/geopackage+sqlite3			gpkg;
 application/gltf-buffer				glbin glbuf;
 application/gml+xml				gml;
 application/gzip				gz tgz;
+application/br                  br;
 application/hyperstudio				stk;
 application/inkml+xml				ink inkml;
 application/ipfix				ipfix;

services/nginx/nginx.conf.template

@@ -1,6 +1,12 @@
 error_log log/error.log error;
 pid run/nginx.pid;
 
+{% if fill_config_feature.nginx_brotli.dynamic and fill_config_feature.nginx_brotli.modules_dir %}
+load_module {{ fill_config_feature.nginx_brotli.modules_dir }}/ngx_http_brotli_filter_module.so; # for compressing responses on-the-fly
+load_module {{ fill_config_feature.nginx_brotli.modules_dir }}/ngx_http_brotli_static_module.so; # for serving pre-compressed files
+{% endif %}
+
+
 # Choose number of NGINX worker processes based on number of CPUs
 worker_processes auto;
 
@@ -21,6 +27,19 @@ http {
              text/javascript
              application/javascript;
 
+  {% if fill_config_feature.nginx_brotli.installed %}
+  # also enable brotli compression
+  brotli on;
+  brotli_comp_level 6;
+  brotli_types text/plain
+               text/css
+               application/json
+               application/x-javascript
+               application/xml
+               text/javascript
+               application/javascript;
+  {% endif %}
+
   # Only retry if there was a communication error, not a timeout
   # on the Tornado server (to avoid propagating "queries of death"
   # to all frontends)

tools/fill_conf_values.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import os
+import subprocess
 
 import jinja2
 from status import status
@@ -36,12 +37,76 @@ def hash_filter(string, htype):
     return h.hexdigest()
 
 
+def nginx_brotli_installed():
+    """Check if the nginx brotli module is installed
+
+    Returns
+    -------
+    installed : bool
+        True if the nginx brotli module is installed, False otherwise
+    dynamic : bool
+        True if the module is dynamically loaded, False otherwise
+    modules_dir : str
+        The directory where the nginx modules are located if dynamic loading is used
+    """
+
+    installed = False
+    dynamic = False
+    dir = None
+
+    try:
+        output = subprocess.check_output(
+            ["nginx", "-V"], stderr=subprocess.STDOUT
+        ).decode("utf-8")
+        if (
+            "--add-module" in output
+            and "brotli" in output.split("--add-module")[1].strip()
+        ):
+            installed = True
+        elif (
+            "--add-dynamic-module" in output
+            and "brotli" in output.split("--add-dynamic-module")[1].strip()
+        ):
+            installed = True
+            dynamic = True
+            # we try to figure out where the modules are, there are 2 possibilities
+            # 1. the --modules-path directive is used
+            # 2. the default directory is used, which is where the configuration file is located so we look for it
+            if "--modules-path" in output:
+                dir = output.split("--modules-path=")[1].split(" ")[0]
+            elif "--conf-path" in output:
+                dir = os.path.dirname(
+                    output.split("--conf-path=")[1].split(" ")[0]
+                ).replace("nginx.conf", "modules")
+            if dir is not None and not os.path.isdir(dir):
+                dir = None
+            if dir is None:
+                print(
+                    "Brotli is installed dynamically, but couldn't find the nginx modules directory. Skipping."
+                )
+                installed = False
+                dynamic = False
+            else:
+                dir = dir.rstrip("/")
+    except subprocess.CalledProcessError:
+        pass
+    return installed, dynamic, dir
+
+
 custom_filters = {"md5sum": md5sum, "version": version, "hash": hash_filter}
 
 
 def fill_config_file_values(template_paths):
     log("Compiling configuration templates")
     env, cfg = load_env()
+    installed, dynamic, modules_dir = nginx_brotli_installed()
+    cfg["fill_config_feature"] = {
+        "nginx_brotli": {
+            "installed": installed,
+            "dynamic": dynamic,
+            "modules_dir": modules_dir,
+        }
+    }
 
     for template_path in template_paths:
         with status(template_path):