dockerfile + compose for ol-www0 web_nginx

docker-compose.production.yml

@@ -57,7 +57,11 @@ services:
 
   covers_nginx:
     profiles: ["ol-covers0"]
-    image: nginx:1.19.4
+    image: "${OLIMAGE:-openlibrary/olbase:latest}"
+    user: root
+    command: docker/ol-nginx-start.sh
+    environment:
+      - CRONTAB_FILES=/etc/cron.d/archive-webserver-logs
     restart: unless-stopped
     depends_on:
       - covers
@@ -69,6 +73,13 @@ services:
       # Needs access to openlibrary for static files
       - ../olsystem:/olsystem
       - /1/var/lib/openlibrary/sitemaps/sitemaps:/sitemaps
+      # web log rotation
+      - ../olsystem/etc/logrotate.d/nginx:/etc/logrotate.d/nginx
+      # Persist the nginx logs
+      - /1/var/log/nginx:/var/log/nginx
+      # Archive nginx logs regularly
+      - ../olsystem/etc/cron.d/archive-webserver-logs:/etc/cron.d/archive-webserver-logs
+      - archive-webserver-logs-data:/archive-webserver-logs-data
     ports:
       - 80:80
       - 443:443
@@ -79,7 +90,6 @@ services:
         max-size: "512m"
         max-file: "4"
     secrets:
-      - petabox_seed
       # Needed by public_nginx.conf
       - ssl_certificate
       - ssl_certificate_key
@@ -116,10 +126,65 @@ services:
       - ../olsystem:/olsystem
       - infobase-writelog:/1/var/lib/openlibrary/infobase/log
       - infobase-errorlog:/1/var/log/openlibrary/infobase-errors
+  web_haproxy:
+    profiles: ["ol-www0"]
+    image: haproxy:2.3.5
+    restart: unless-stopped
+    hostname: "$HOSTNAME"
+    networks:
+      - webnet
+    volumes:
+      - ../olsystem/etc/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+    ports:
+      - 7072:7072
+    logging:
+      options:
+        max-size: "512m"
+        max-file: "4"
+  web_nginx:
+    profiles: ["ol-www0"]
+    restart: unless-stopped
+    hostname: "ol-www0" # always want this hostname (or nginx backups will be misnamed)
+    image: "${OLIMAGE:-openlibrary/olbase:latest}"
+    user: root
+    command: docker/ol-nginx-start.sh
+    environment:
+      - CRONTAB_FILES=/etc/cron.d/archive-webserver-logs
+    volumes:
+      # nginx configurations
+      - ./docker/nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./docker/web_nginx.conf:/etc/nginx/sites-enabled/openlibrary.conf:ro
+      # Needed for HTTPS, since this is a public server
+      - ./docker/public_nginx.conf:/etc/nginx/sites-available/public_nginx.conf:ro
+      # archive web log uploads
+      - ../olsystem:/olsystem
+      # web log rotation
+      - ../olsystem/etc/logrotate.d/nginx:/etc/logrotate.d/nginx
+      # Persist the nginx logs
+      - /1/var/log/nginx:/var/log/nginx
+      # sitemap generation (we also need olsystem/etc/cron.d + openlibrary/scripts)
+      - /1/var/lib/openlibrary/sitemaps/sitemaps:/sitemaps
+      # Archive nginx logs regularly
+      - ../olsystem/etc/cron.d/archive-webserver-logs:/etc/cron.d/archive-webserver-logs
+      - archive-webserver-logs-data:/archive-webserver-logs-data
+    ports:
+      - 80:80
+      - 443:443
+    networks:
+      - webnet
+    logging:
+      options:
+        max-size: "512m"
+        max-file: "4"
+    secrets:
+      - ssl_certificate
+      - ssl_certificate_key
 
   infobase_nginx:
     profiles: ["ol-home0"]
-    image: nginx:1.19.4
+    image: "${OLIMAGE:-openlibrary/olbase:latest}"
+    user: root
+    command: docker/ol-nginx-start.sh
     restart: unless-stopped
     depends_on:
       - infobase
@@ -128,12 +193,12 @@ services:
       - ./docker/infobase_nginx.conf:/etc/nginx/sites-enabled/infobase_nginx.conf:ro
       # Needs olsystem for black-listed IPs
       - ../olsystem:/olsystem
+      # Log rotation
+      - ../olsystem/etc/logrotate.d/nginx:/etc/logrotate.d/nginx
     ports:
       - 7000:7000
     networks:
       - webnet
-    secrets:
-      - petabox_seed
 
   affiliate-server:
     profiles: ["ol-home0"]
@@ -146,7 +211,7 @@ services:
     ports:
       - 31337:31337
     volumes:
-      -  /opt/olsystem/etc/openlibrary.yml:/openlibrary.yml
+      - /opt/olsystem/etc/openlibrary.yml:/openlibrary.yml
     networks:
       - webnet
     logging:
@@ -162,7 +227,8 @@ services:
     environment:
       - OL_CONFIG=/olsystem/etc/openlibrary.yml
       - OL_URL=https://openlibrary.org/
-      - EXTRA_OPTS=--solr-url http://ol-solr0:8984/solr/openlibrary --no-solr-next
+      - EXTRA_OPTS=--solr-url http://ol-solr0:8984/solr/openlibrary
+        --no-solr-next
     volumes:
       - ../olsystem:/olsystem
     logging:
@@ -212,10 +278,9 @@ services:
       - webnet
       - dbnet
 
+# secrets mounted to /run/secrets/
+# e.g. /run/secrets/ia_db_pw_file
 secrets:
-    petabox_seed:
-      file: /opt/.petabox/seed
-
     ia_db_pw_file:
       file: /opt/.petabox/dbserver
 
@@ -227,3 +292,4 @@ secrets:
 volumes:
     infobase-writelog:
     infobase-errorlog:
+    archive-webserver-logs-data:

docker/Dockerfile.olbase

@@ -46,6 +46,23 @@ RUN apt-get -qq update && apt-get install -y build-essential libssl-dev zlib1g-d
     libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
     xz-utils tk-dev libffi-dev liblzma-dev python-openssl git
 
+# Install Archive.org nginx w/ IP anonymization
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends nginx curl \
+    # nginx-plus
+    apt-transport-https lsb-release ca-certificates wget \
+    # log rotation service for ol-nginx
+    logrotate
+RUN wget -O - https://openresty.org/package/pubkey.gpg | apt-key add -
+RUN echo "deb http://openresty.org/package/ubuntu $(lsb_release -sc) main" \
+    | tee /etc/apt/sources.list.d/openresty.list
+RUN apt-get update && apt-get -y install --no-install-recommends openresty
+RUN rm /usr/sbin/nginx
+RUN curl -L https://archive.org/download/nginx/nginx -o /usr/sbin/nginx
+RUN chmod +x /usr/sbin/nginx
+# Remove the stock nginx config file
+RUN rm /etc/nginx/sites-enabled/default
+
 # Install latest pyenv (https://github.com/pyenv/pyenv-installer)
 USER openlibrary
 RUN curl https://pyenv.run | bash && \

docker/covers_nginx.conf

@@ -2,7 +2,6 @@ server {
       include /etc/nginx/sites-available/public_nginx.conf;
       server_name  covers.openlibrary.org;
 
-      include /run/secrets/petabox_seed;
       root /openlibrary;
 
       keepalive_timeout 5;

docker/infobase_nginx.conf

@@ -5,7 +5,6 @@ server {
     # Required when querying for multiple infobase documents at once.
     large_client_header_buffers 4 160k;
 
-    include /run/secrets/petabox_seed;
     root /openlibrary;
 
     location / {

docker/nginx.conf

@@ -1,4 +1,4 @@
-user  nginx;
+user  www-data;
 
 # XXX-Anand: Oct 2013
 # Increased the worker_processes to allow more workers to share the load of https.
@@ -25,7 +25,7 @@ http {
     server_names_hash_bucket_size   64;
     types_hash_bucket_size 64;
 
-    log_format iacombined '$seed$remote_addr $host $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time';
+    log_format iacombined '$remote_addr_ipscrub $host $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time';
     access_log    /var/log/nginx/access.log iacombined;
 
     client_max_body_size 50m;

docker/ol-nginx-start.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+if [ -n "$CRONTAB_FILES" ] ; then
+  crontab $CRONTAB_FILES
+  service cron start
+fi
+
+# logrotate comes from olsystem which is volume mounted
+# logrotate requires files to be 644
+# expect conflicts writing to file
+chmod 644 /etc/logrotate.d/nginx
+logrotate --verbose /etc/logrotate.d/nginx
+
+nginx -g "daemon off;"

docker/web_nginx.conf

@@ -0,0 +1,94 @@
+
+## Gio and Sam
+# added an haproxy to better balance the requests
+# between the webnodes
+# http://www.openlibrary.org/admin?stats
+#
+#
+# old conf (requests balanced by nginx)
+#upstream webnodes {
+#    server ol-web1.us.archive.org:8080;
+#    server ol-web2.us.archive.org:8080;
+#}
+#
+# using haproxy
+upstream webnodes {
+    server web_haproxy:7072;
+}
+
+server {
+    include /etc/nginx/sites-available/public_nginx.conf;
+    server_name  openlibrary.org;
+
+    # Set the referrer policy so browsers send referrers to our own servers
+    # In July 2020, Chrome changed its default referrer policy so any cross-origin
+    # requests only sent the root referrer `/`. Since openlibrary.org
+    # has a different "origin" than analytics.archive.org, the full referrer
+    # path is not sent. This changes the behavior back to the pre-July 2020 change.
+    add_header Referrer-Policy "no-referrer-when-downgrade";
+
+    root        /openlibrary;
+
+    # Show closed-library page on errors.
+    error_page 502 /static/status-500.html;
+    error_page 500 /static/status-500.html;
+
+    # Anand - Oct 2013
+    # Redirect all http URLs except the API calls (ending with .json or /api/*) to https
+    set $api_call "$scheme:noapi";
+    if ($uri ~ '\.json$') {
+        set $api_call "$scheme:api";
+    }
+    if ($uri ~ '^/api/.*$') {
+        set $api_call "$scheme:api";
+    }
+    if ($api_call = "http:noapi") {
+        rewrite ^(.*)$ https://$http_host$1 last;
+    }
+    location / {
+        proxy_pass http://webnodes;
+        proxy_set_header Host $http_host;
+
+        # Gunicorn takes IP from this header
+        proxy_set_header X-Forwarded-For $remote_addr;
+
+        # Hack to make the app pick the right url scheme even when the
+        # app server is http only.
+        proxy_set_header X-Scheme $scheme;
+    }
+
+    location ~ ^/(images/.*|favicon.ico|robots.txt)$ {
+        rewrite ^(.*)$ /static/$1 last;
+    }
+
+    location ~ ^/(y_key_[0-9a-f]+.html|google[0-9a-f]+.html|LiveSearchSiteAuth.xml)$ {
+        root /olsystem/www;
+    }
+
+    location ~ ^/static/(docs|tour|sitemaps|jsondumps|images/shelfview|sampledump.txt.gz)(/.*)?$ {
+        root /sitemaps;
+        autoindex on;
+        rewrite ^/static/(.*)$ /$1 break;
+    }
+
+    location /static {
+       autoindex on;
+       expires 1h;
+    }
+
+    location /static/build {
+        expires max;
+    }
+
+    location /index {
+        root /sitemaps;
+        autoindex on;
+    }
+}
+
+server {
+    include /etc/nginx/sites-available/public_nginx.conf;
+    server_name www.openlibrary.org *.openlibrary.org;
+
+    rewrite ^(.*)$ http://openlibrary.org$1 permanent;
+}