From a3596372f16ac3d46540198b1ad4f500755c01de Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Mon, 16 Dec 2024 12:45:39 -0500 Subject: [PATCH 1/8] Use NJS for nginx IP anonymization --- docker/nginx.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/nginx.conf b/docker/nginx.conf index 71093e6613f..65a1c662578 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -25,7 +25,8 @@ http { server_names_hash_bucket_size 64; types_hash_bucket_size 64; - log_format iacombined '$remote_addr_ipscrub $host $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time'; + # Logging / IP Anonymization + include /olsystem/etc/nginx/logging.conf; access_log /var/log/nginx/access.log iacombined; client_max_body_size 50m; From 5b301dd9afbb25efc80cbc7e35ed6fbfa01878f5 Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Mon, 16 Dec 2024 13:00:39 -0500 Subject: [PATCH 2/8] Switch to stock nginx with NJS support --- docker/Dockerfile.olbase | 17 ++++++----------- docker/nginx.conf | 3 +++ scripts/install_openresty.sh | 14 -------------- 3 files changed, 9 insertions(+), 25 deletions(-) delete mode 100755 scripts/install_openresty.sh diff --git a/docker/Dockerfile.olbase b/docker/Dockerfile.olbase index e5aea47622a..06a2530f194 100644 --- a/docker/Dockerfile.olbase +++ b/docker/Dockerfile.olbase @@ -38,22 +38,17 @@ RUN apt-get -qq update && apt-get install -y \ COPY scripts/install_nodejs.sh ./ RUN ./install_nodejs.sh && rm ./install_nodejs.sh -# Install Archive.org nginx w/ IP anonymization +# Install nginx USER root RUN apt-get update && apt-get install -y --no-install-recommends nginx curl letsencrypt \ - # nginx-plus - apt-transport-https lsb-release ca-certificates wget \ # log rotation service for ol-nginx logrotate \ # rsync service for pulling monthly sitemaps from ol-home0 to ol-www0 - rsync -COPY scripts/install_openresty.sh ./ -RUN ./install_openresty.sh && rm ./install_openresty.sh -RUN rm /usr/sbin/nginx -RUN curl -L https://archive.org/download/nginx/nginx -o /usr/sbin/nginx -RUN chmod +x /usr/sbin/nginx -# Remove the stock nginx config file -RUN rm /etc/nginx/sites-enabled/default + rsync \ + # NJS for IP anonymization + libnginx-mod-http-js \ + # Remove the stock nginx config file + && rm /etc/nginx/sites-enabled/default RUN mkdir -p /var/log/openlibrary /var/lib/openlibrary && chown openlibrary:openlibrary /var/log/openlibrary /var/lib/openlibrary \ && mkdir /openlibrary && chown openlibrary:openlibrary /openlibrary \ diff --git a/docker/nginx.conf b/docker/nginx.conf index 65a1c662578..aafb34bf80c 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -1,3 +1,6 @@ +# Needed for IP anonymization +load_module modules/ngx_http_js_module.so; + user www-data; # XXX-Anand: Oct 2013 diff --git a/scripts/install_openresty.sh b/scripts/install_openresty.sh deleted file mode 100755 index b8e5751ed73..00000000000 --- a/scripts/install_openresty.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -machine=$(uname -m) - -if [[ "${machine}" == "aarch64" || "${machine}" == "arm64" ]]; then - echo "Running on ARM64 architecture (e.g., Apple M1)" - echo "openresty still doesn't work on arm see https://github.com/openresty/openresty/issues/840 and \ - https://github.com/internetarchive/openlibrary/issues/6316" -else - wget -O - https://openresty.org/package/pubkey.gpg | apt-key add - - echo "deb http://openresty.org/package/debian $(lsb_release -sc) openresty" \ - | tee /etc/apt/sources.list.d/openresty.list - apt-get update && apt-get -y install --no-install-recommends openresty -fi From 60e593d66af18670fb1120761a16ae52af5d566f Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Mon, 16 Dec 2024 16:17:01 -0500 Subject: [PATCH 3/8] Use newer version of NJS --- docker/Dockerfile.olbase | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.olbase b/docker/Dockerfile.olbase index 06a2530f194..7e460c161b8 100644 --- a/docker/Dockerfile.olbase +++ b/docker/Dockerfile.olbase @@ -40,14 +40,20 @@ RUN ./install_nodejs.sh && rm ./install_nodejs.sh # Install nginx USER root -RUN apt-get update && apt-get install -y --no-install-recommends nginx curl letsencrypt \ +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ # log rotation service for ol-nginx logrotate \ # rsync service for pulling monthly sitemaps from ol-home0 to ol-www0 rsync \ - # NJS for IP anonymization - libnginx-mod-http-js \ - # Remove the stock nginx config file + # NJS for IP anonymization + && curl -fsSL https://nginx.org/keys/nginx_signing.key | tee /usr/share/keyrings/nginx-keyring.asc \ + && echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.asc] http://nginx.org/packages/ubuntu $(lsb_release -cs) nginx" \ + > /etc/apt/sources.list.d/nginx.list \ + && apt-get update \ + # Install nginx and the NJS module + && apt-get install -y --no-install-recommends nginx nginx-module-njs letsencrypt \ + # Remove the stock nginx config file && rm /etc/nginx/sites-enabled/default RUN mkdir -p /var/log/openlibrary /var/lib/openlibrary && chown openlibrary:openlibrary /var/log/openlibrary /var/lib/openlibrary \ From 76b6289ecef058ec26b74a05779f394a5a902f94 Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Mon, 16 Dec 2024 16:21:59 -0500 Subject: [PATCH 4/8] fixup! Use newer version of NJS --- docker/Dockerfile.olbase | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.olbase b/docker/Dockerfile.olbase index 7e460c161b8..a661d9f0b25 100644 --- a/docker/Dockerfile.olbase +++ b/docker/Dockerfile.olbase @@ -46,9 +46,11 @@ RUN apt-get update \ logrotate \ # rsync service for pulling monthly sitemaps from ol-home0 to ol-www0 rsync \ - # NJS for IP anonymization + # Add the NGINX signing key && curl -fsSL https://nginx.org/keys/nginx_signing.key | tee /usr/share/keyrings/nginx-keyring.asc \ - && echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.asc] http://nginx.org/packages/ubuntu $(lsb_release -cs) nginx" \ + # Add the NGINX repository for Debian + && echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.asc] http://nginx.org/packages/debian $(lsb_release -cs) nginx" \ + > /etc/apt/sources.list.d/nginx.list || echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.asc] http://nginx.org/packages/debian bullseye nginx" \ > /etc/apt/sources.list.d/nginx.list \ && apt-get update \ # Install nginx and the NJS module From 3917c38bd84fbd0e547c1d2fbe823b7e99729fa7 Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Mon, 16 Dec 2024 16:27:20 -0500 Subject: [PATCH 5/8] fixup! Use newer version of NJS --- docker/Dockerfile.olbase | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile.olbase b/docker/Dockerfile.olbase index a661d9f0b25..982dceae596 100644 --- a/docker/Dockerfile.olbase +++ b/docker/Dockerfile.olbase @@ -50,7 +50,6 @@ RUN apt-get update \ && curl -fsSL https://nginx.org/keys/nginx_signing.key | tee /usr/share/keyrings/nginx-keyring.asc \ # Add the NGINX repository for Debian && echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.asc] http://nginx.org/packages/debian $(lsb_release -cs) nginx" \ - > /etc/apt/sources.list.d/nginx.list || echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.asc] http://nginx.org/packages/debian bullseye nginx" \ > /etc/apt/sources.list.d/nginx.list \ && apt-get update \ # Install nginx and the NJS module From f15914321f6318d4c599bc804a7e6689e68fbf2b Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Mon, 16 Dec 2024 16:35:13 -0500 Subject: [PATCH 6/8] fixup! Use newer version of NJS --- docker/Dockerfile.olbase | 18 ++---------------- scripts/install_nginx.sh | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 16 deletions(-) create mode 100755 scripts/install_nginx.sh diff --git a/docker/Dockerfile.olbase b/docker/Dockerfile.olbase index 982dceae596..2d2be3fb2e3 100644 --- a/docker/Dockerfile.olbase +++ b/docker/Dockerfile.olbase @@ -40,22 +40,8 @@ RUN ./install_nodejs.sh && rm ./install_nodejs.sh # Install nginx USER root -RUN apt-get update \ - && apt-get install -y --no-install-recommends curl \ - # log rotation service for ol-nginx - logrotate \ - # rsync service for pulling monthly sitemaps from ol-home0 to ol-www0 - rsync \ - # Add the NGINX signing key - && curl -fsSL https://nginx.org/keys/nginx_signing.key | tee /usr/share/keyrings/nginx-keyring.asc \ - # Add the NGINX repository for Debian - && echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.asc] http://nginx.org/packages/debian $(lsb_release -cs) nginx" \ - > /etc/apt/sources.list.d/nginx.list \ - && apt-get update \ - # Install nginx and the NJS module - && apt-get install -y --no-install-recommends nginx nginx-module-njs letsencrypt \ - # Remove the stock nginx config file - && rm /etc/nginx/sites-enabled/default +COPY scripts/install_nginx.sh ./ +RUN ./install_nginx.sh && rm ./install_nginx.sh RUN mkdir -p /var/log/openlibrary /var/lib/openlibrary && chown openlibrary:openlibrary /var/log/openlibrary /var/lib/openlibrary \ && mkdir /openlibrary && chown openlibrary:openlibrary /openlibrary \ diff --git a/scripts/install_nginx.sh b/scripts/install_nginx.sh new file mode 100755 index 00000000000..5328d6d21b4 --- /dev/null +++ b/scripts/install_nginx.sh @@ -0,0 +1,21 @@ +#! /bin/bash + +apt-get update + +# log rotation service for ol-nginx +# rsync service for pulling monthly sitemaps from ol-home0 to ol-www0 +apt-get install -y --no-install-recommends curl \ + logrotate \ + rsync + +# Add the NGINX signing key + Repo +curl -fsSL https://nginx.org/keys/nginx_signing.key | tee /usr/share/keyrings/nginx-keyring.asc +echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.asc] http://nginx.org/packages/debian $(lsb_release -cs) nginx" \ + > /etc/apt/sources.list.d/nginx.list + +# Install nginx and the NJS module +apt-get update +apt-get install -y --no-install-recommends nginx nginx-module-njs letsencrypt + +# Remove the stock nginx config file +rm /etc/nginx/sites-enabled/default \ No newline at end of file From 9ca1acc8483cf3d4ccae8e3c65d3525104ec3c96 Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Mon, 16 Dec 2024 16:38:46 -0500 Subject: [PATCH 7/8] fixup! Use newer version of NJS --- scripts/install_nginx.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/install_nginx.sh b/scripts/install_nginx.sh index 5328d6d21b4..1af497a5e86 100755 --- a/scripts/install_nginx.sh +++ b/scripts/install_nginx.sh @@ -6,7 +6,8 @@ apt-get update # rsync service for pulling monthly sitemaps from ol-home0 to ol-www0 apt-get install -y --no-install-recommends curl \ logrotate \ - rsync + rsync \ + lsb-release # Add the NGINX signing key + Repo curl -fsSL https://nginx.org/keys/nginx_signing.key | tee /usr/share/keyrings/nginx-keyring.asc From c269882f29f2aaf7b9ee291765969ae00b2d3d92 Mon Sep 17 00:00:00 2001 From: Drini Cami Date: Mon, 16 Dec 2024 16:41:36 -0500 Subject: [PATCH 8/8] fixup! Use newer version of NJS --- scripts/install_nginx.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/install_nginx.sh b/scripts/install_nginx.sh index 1af497a5e86..8f4f81378ae 100755 --- a/scripts/install_nginx.sh +++ b/scripts/install_nginx.sh @@ -17,6 +17,3 @@ echo "deb [signed-by=/usr/share/keyrings/nginx-keyring.asc] http://nginx.org/pac # Install nginx and the NJS module apt-get update apt-get install -y --no-install-recommends nginx nginx-module-njs letsencrypt - -# Remove the stock nginx config file -rm /etc/nginx/sites-enabled/default \ No newline at end of file