Merge pull request #264 from lotooo/master

chore: Make redis, postgres and rabbitmq hosts configurable via env variables
metabrainz · Mar 21, 2024 · 82f54d7 · 82f54d7
2 parents 4984957 + e1a83a8
commit 82f54d7
Show file tree

Hide file tree

Showing 20 changed files with 83 additions and 35 deletions.
diff --git a/README.md b/README.md
@@ -401,6 +401,40 @@ This can be changed by creating a custom configuration file under
 [and finally](https://docs.docker.com/storage/bind-mounts/#choose-the--v-or---mount-flag)
 setting the Docker environment variable `SIR_CONFIG_PATH` to its path.
 
+#### Customize backend Postgres server
+
+By default, the services `indexer` and `musicbrainz` are trying to connect to the host `db` (for both read-only and write host) but the hosts can
+be customized using the `MUSICBRAINZ_POSTGRES_SERVER` and `MUSICBRAINZ_POSTGRES_READONLY_SERVER` environment variables.
+
+Notes:
+* After switching to another Postgres server:
+  * If not transferring data, it is needed to create the database again.
+  * For live indexing, the RabbitMQ server has to still be reachable from the Postgres server.
+* The helper scripts `check-search-indexes` and `create-amqp-extension` won’t work anymore.
+* The service `db` will still be up even if unused.
+
+#### Customize backend RabbitMQ server
+
+By default, the services `db`, `indexer` and `musicbrainz` are trying to connect to the host `mq`
+but the host can be customized using the `MUSICBRAINZ_RABBITMQ_SERVER` environment variable.
+
+Notes:
+* After switching to another RabbitMQ server:
+  - Live indexing requires to go through AMQP Setup again.
+  - If not transferring data, it might be needed to build search indexes again.
+* The helper script `purge-message-queues` won’t work anymore.
+* The service `mq` will still be up even if unused.
+
+#### Customize backend Redis server
+
+By default, the service `musicbrainz` is trying to connect to the host `redis`
+but the host can be customized using the `MUSICBRAINZ_REDIS_SERVER` environment variable.
+
+Notes:
+* After switching to another Redis server:
+  - If not transferring data, MusicBrainz user sessions will be reset.
+* The service `redis` will still be running even if unused.
+
 ### Docker Compose overrides
 
 In Docker Compose, it is possible to override the base configuration using

diff --git a/build/musicbrainz-dev/DBDefs.pm b/build/musicbrainz-dev/DBDefs.pm
@@ -54,15 +54,15 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases(
         database    => "musicbrainz_db",
         username    => "$ENV{POSTGRES_USER}",
         password    => "$ENV{POSTGRES_PASSWORD}",
-        host        => "db",
+        host        => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}",
         port        => "5432",
     },
     # How to connect to a test database
     TEST => {
         database    => "musicbrainz_test",
         username    => "$ENV{POSTGRES_USER}",
         password    => "$ENV{POSTGRES_PASSWORD}",
-        host        => "db",
+        host        => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}",
         port        => "5432",
     },
     # How to connect to a Selenium test database. This database is created
@@ -73,15 +73,15 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases(
         schema      => 'musicbrainz',
         username    => "$ENV{POSTGRES_USER}",
         password    => "$ENV{POSTGRES_PASSWORD}",
-        host        => "db",
+        host        => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}",
         port        => "5432",
     },
     # How to connect for read-only access.  See "REPLICATION_TYPE" (below)
     READONLY => {
         database    => "musicbrainz_db",
         username    => "$ENV{POSTGRES_USER}",
         password    => "$ENV{POSTGRES_PASSWORD}",
-        host        => "db",
+        host        => "$ENV{MUSICBRAINZ_POSTGRES_READONLY_SERVER}",
         port        => "5432",
     },
     # How to connect for read-only access to the production database in standby mode.
@@ -101,7 +101,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases(
         database    => "template1",
         username    => "$ENV{POSTGRES_USER}",
         password    => "$ENV{POSTGRES_PASSWORD}",
-        host        => "db",
+        host        => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}",
         port        => "5432",
     },
     # How to connect when running maintenance scripts located under admin/.
@@ -111,7 +111,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases(
 #       database    => "musicbrainz_db",
 #       username    => "$ENV{POSTGRES_USER}",
 #       password    => "$ENV{POSTGRES_PASSWORD}",
-#       host        => "db",
+#       host        => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}",
 #       port        => "5432",
 #   },
     # Fill out only if RAWDATA lives on a different host from the READWRITE server.
@@ -274,7 +274,7 @@ sub PLUGIN_CACHE_OPTIONS {
     my $self = shift;
     return {
         class => 'MusicBrainz::Server::CacheWrapper::Redis',
-        server => 'redis:6379',
+        server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379",
         namespace => $self->CACHE_NAMESPACE . 'Catalyst:',
     };
 }
@@ -289,7 +289,7 @@ sub CACHE_MANAGER_OPTIONS {
             external => {
                 class => 'MusicBrainz::Server::CacheWrapper::Redis',
                 options => {
-                    server => 'redis:6379',
+                    server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379",
                     namespace => $self->CACHE_NAMESPACE,
                 },
             },
@@ -328,7 +328,7 @@ sub DATASTORE_REDIS_ARGS {
     return {
         database => 0,
         namespace => $self->CACHE_NAMESPACE,
-        server => 'redis:6379',
+        server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379",
         test_database => 1,
     };
 }

diff --git a/build/musicbrainz-dev/Dockerfile b/build/musicbrainz-dev/Dockerfile
@@ -86,6 +86,10 @@ ENV BASH_ENV=/noninteractive.bash_env \
     MUSICBRAINZ_BASE_DOWNLOAD_URL=https://data.metabrainz.org/pub/musicbrainz \
     MUSICBRAINZ_CATALYST_DEBUG=0 \
     MUSICBRAINZ_DEVELOPMENT_SERVER=1 \
+    MUSICBRAINZ_POSTGRES_SERVER=db \
+    MUSICBRAINZ_POSTGRES_READONLY_SERVER=db \
+    MUSICBRAINZ_RABBITMQ_SERVER=mq \
+    MUSICBRAINZ_REDIS_SERVER=redis \
     MUSICBRAINZ_SEARCH_SERVER=search:8983/solr \
     MUSICBRAINZ_SERVER_PROCESSES=1 \
     MUSICBRAINZ_STANDALONE_SERVER=1 \

diff --git a/build/musicbrainz-dev/scripts/createdb.sh b/build/musicbrainz-dev/scripts/createdb.sh
@@ -94,7 +94,7 @@ for F in "${DUMP_FILES[@]}"; do
 done
 
 echo "found existing dumps"
-dockerize -wait tcp://db:5432 -timeout 60s sleep 0
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0
 
 update-perl.sh
 

diff --git a/build/musicbrainz-dev/scripts/indexer-triggers.sh b/build/musicbrainz-dev/scripts/indexer-triggers.sh
@@ -12,7 +12,7 @@ fi
 
 INDEXER_SQL_DIR="$1"
 
-dockerize -wait tcp://db:5432 -timeout 60s sleep 0
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0
 
 cd /musicbrainz-server
 

diff --git a/build/musicbrainz-dev/scripts/recreatedb.sh b/build/musicbrainz-dev/scripts/recreatedb.sh
@@ -2,5 +2,5 @@
 
 set -e
 
-dockerize -wait tcp://db:5432 -timeout 60s sleep 0
-psql postgres -U musicbrainz -h db -c "DROP DATABASE IF EXISTS musicbrainz_db;"; createdb.sh "$@"
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0
+psql postgres -U musicbrainz -h "${MUSICBRAINZ_POSTGRES_SERVER}" -c "DROP DATABASE IF EXISTS musicbrainz_db;"; createdb.sh "$@"
diff --git a/build/musicbrainz-dev/scripts/replication.sh b/build/musicbrainz-dev/scripts/replication.sh
@@ -2,5 +2,5 @@
 
 set -e
 
-dockerize -wait tcp://db:5432 -timeout 60s sleep 0
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0
 exec /musicbrainz-server/admin/cron/mirror.sh
diff --git a/build/musicbrainz-dev/scripts/update-javascript.sh b/build/musicbrainz-dev/scripts/update-javascript.sh
@@ -6,6 +6,6 @@ cd /musicbrainz-server
 
 yarn
 
-dockerize -wait tcp://db:5432 -timeout 60s -wait tcp://mq:5672 -timeout 60s -wait tcp://redis:6379 -timeout 60s
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s -wait "tcp://${MUSICBRAINZ_RABBITMQ_SERVER}:5672" -timeout 60s -wait "tcp://${MUSICBRAINZ_REDIS_SERVER}:6379" -timeout 60s
 
 ./script/compile_resources.sh --watch client server tests &> /compile_resources.log &
diff --git a/build/musicbrainz/DBDefs.pm b/build/musicbrainz/DBDefs.pm
@@ -54,7 +54,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases(
         database    => "musicbrainz_db",
         username    => "$ENV{POSTGRES_USER}",
         password    => "$ENV{POSTGRES_PASSWORD}",
-        host        => "db",
+        host        => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}",
         port        => "5432",
     },
     # How to connect to a test database
@@ -80,15 +80,15 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases(
         database    => "musicbrainz_db",
         username    => "$ENV{POSTGRES_USER}",
         password    => "$ENV{POSTGRES_PASSWORD}",
-        host        => "db",
+        host        => "$ENV{MUSICBRAINZ_POSTGRES_READONLY_SERVER}",
         port        => "5432",
     },
     # How to connect for administrative access
     SYSTEM    => {
         database    => "template1",
         username    => "$ENV{POSTGRES_USER}",
         password    => "$ENV{POSTGRES_PASSWORD}",
-        host        => "db",
+        host        => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}",
         port        => "5432",
     },
     # How to connect when running maintenance scripts located under admin/.
@@ -261,7 +261,7 @@ sub PLUGIN_CACHE_OPTIONS {
     my $self = shift;
     return {
         class => 'MusicBrainz::Server::CacheWrapper::Redis',
-        server => 'redis:6379',
+        server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379",
         namespace => $self->CACHE_NAMESPACE . 'Catalyst:',
     };
 }
@@ -276,7 +276,7 @@ sub CACHE_MANAGER_OPTIONS {
             external => {
                 class => 'MusicBrainz::Server::CacheWrapper::Redis',
                 options => {
-                    server => 'redis:6379',
+                    server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379",
                     namespace => $self->CACHE_NAMESPACE,
                 },
             },
@@ -315,7 +315,7 @@ sub DATASTORE_REDIS_ARGS {
     return {
         database => 0,
         namespace => $self->CACHE_NAMESPACE,
-        server => 'redis:6379',
+        server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379",
         test_database => 1,
     };
 }

diff --git a/build/musicbrainz/Dockerfile b/build/musicbrainz/Dockerfile
@@ -87,6 +87,10 @@ ENV BASH_ENV=/noninteractive.bash_env \
     MUSICBRAINZ_CATALYST_DEBUG=0 \
     MUSICBRAINZ_DB_SCHEMA_SEQUENCE=28 \
     MUSICBRAINZ_DEVELOPMENT_SERVER=0 \
+    MUSICBRAINZ_POSTGRES_SERVER=db \
+    MUSICBRAINZ_POSTGRES_READONLY_SERVER=db \
+    MUSICBRAINZ_RABBITMQ_SERVER=mq \
+    MUSICBRAINZ_REDIS_SERVER=redis \
     MUSICBRAINZ_SEARCH_SERVER=search:8983/solr \
     MUSICBRAINZ_SERVER_PROCESSES=10 \
     MUSICBRAINZ_STANDALONE_SERVER=0 \

diff --git a/build/musicbrainz/scripts/createdb.sh b/build/musicbrainz/scripts/createdb.sh
@@ -94,7 +94,7 @@ for F in "${DUMP_FILES[@]}"; do
 done
 
 echo "found existing dumps"
-dockerize -wait tcp://db:5432 -timeout 60s sleep 0
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0
 
 mkdir -p $TMP_DIR
 cd /media/dbdump

diff --git a/build/musicbrainz/scripts/indexer-triggers.sh b/build/musicbrainz/scripts/indexer-triggers.sh
@@ -12,7 +12,7 @@ fi
 
 INDEXER_SQL_DIR="$1"
 
-dockerize -wait tcp://db:5432 -timeout 60s sleep 0
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0
 
 cd /musicbrainz-server
 

diff --git a/build/musicbrainz/scripts/load-crontab-only.sh b/build/musicbrainz/scripts/load-crontab-only.sh
@@ -3,8 +3,8 @@
 set -e -u
 
 dockerize \
-  -wait tcp://db:5432 -timeout 60s \
-  -wait tcp://redis:6379 -timeout 60s \
+  -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s \
+  -wait "tcp://${MUSICBRAINZ_REDIS_SERVER}:6379" -timeout 60s \
   true
 
 if [ -f /crons.conf -a -s /crons.conf ]

diff --git a/build/musicbrainz/scripts/recreatedb.sh b/build/musicbrainz/scripts/recreatedb.sh
@@ -2,5 +2,5 @@
 
 set -e
 
-dockerize -wait tcp://db:5432 -timeout 60s sleep 0
-psql postgres -U musicbrainz -h db -c "DROP DATABASE IF EXISTS musicbrainz_db;"; createdb.sh "$@"
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0
+psql postgres -U musicbrainz -h "${MUSICBRAINZ_POSTGRES_SERVER}" -c "DROP DATABASE IF EXISTS musicbrainz_db;"; createdb.sh "$@"
diff --git a/build/musicbrainz/scripts/replication.sh b/build/musicbrainz/scripts/replication.sh
@@ -2,5 +2,5 @@
 
 set -e
 
-dockerize -wait tcp://db:5432 -timeout 60s sleep 0
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0
 exec /musicbrainz-server/admin/cron/mirror.sh
diff --git a/build/musicbrainz/scripts/start.sh b/build/musicbrainz/scripts/start.sh
@@ -9,7 +9,7 @@ then
   /musicbrainz-server/script/compile_resources.sh
 fi
 
-dockerize -wait tcp://db:5432 -timeout 60s -wait tcp://mq:5672 -timeout 60s -wait tcp://redis:6379 -timeout 60s start_mb_renderer.pl
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s -wait "tcp://${MUSICBRAINZ_RABBITMQ_SERVER}:5672" -timeout 60s -wait "tcp://${MUSICBRAINZ_REDIS_SERVER}:6379" -timeout 60s start_mb_renderer.pl
 
 if [ -f /crons.conf -a -s /crons.conf ]
 then

diff --git a/build/musicbrainz/scripts/upgrade-db-schema.sh b/build/musicbrainz/scripts/upgrade-db-schema.sh
@@ -4,4 +4,4 @@ set -e -u
 
 export MUSICBRAINZ_DB_SCHEMA_SEQUENCE=27
 
-dockerize -wait tcp://db:5432 -timeout 60s ./upgrade.sh
+dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s ./upgrade.sh
diff --git a/build/sir-dev/scripts/docker-entrypoint.sh b/build/sir-dev/scripts/docker-entrypoint.sh
@@ -12,8 +12,8 @@ pip install \
   -r requirements_dev.txt
 
 dockerize \
-  -wait tcp://db:5432 -timeout 60s \
-  -wait tcp://mq:5672 -timeout 60s \
+  -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s \
+  -wait "tcp://${MUSICBRAINZ_RABBITMQ_SERVER}:5672" -timeout 60s \
   -wait tcp://search:8983 -timeout 60s \
   sleep 0
 

diff --git a/build/sir/Dockerfile b/build/sir/Dockerfile
@@ -46,5 +46,11 @@ RUN git clone --depth=1 --branch "v${SIR_VERSION}" https://github.com/metabrainz
 
 WORKDIR /code
 
+ENV MUSICBRAINZ_RABBITMQ_SERVER=mq \
+    MUSICBRAINZ_POSTGRES_SERVER=db \
+    MUSICBRAINZ_POSTGRES_READONLY_SERVER=db \
+    MUSICBRAINZ_SEARCH_SERVER=search:8983/solr
+
 ENV POSTGRES_USER=musicbrainz
 ENV POSTGRES_PASSWORD=musicbrainz
+
diff --git a/default/indexer.ini b/default/indexer.ini
@@ -1,12 +1,12 @@
 [database]
 dbname = musicbrainz_db
-host = db
+host = ${MUSICBRAINZ_POSTGRES_SERVER}
 password = ${POSTGRES_PASSWORD}
 port = 5432
 user = ${POSTGRES_USER}
 
 [solr]
-uri = http://search:8983/solr
+uri = ${MUSICBRAINZ_SEARCH_SERVER}
 batch_size = 200
 
 [sir]
@@ -18,7 +18,7 @@ query_batch_size = 5000
 wscompat = on
 
 [rabbitmq]
-host = mq
+host = ${MUSICBRAINZ_RABBITMQ_SERVER}
 user = sir
 password = sir
 vhost = /search-index-rebuilder
Original file line number	Diff line number	Diff line change
Expand Up		@@ -4,4 +4,4 @@ set -e -u

		export MUSICBRAINZ_DB_SCHEMA_SEQUENCE=27

		dockerize -wait tcp://db:5432 -timeout 60s ./upgrade.sh
		dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s ./upgrade.sh