diff --git a/README.md b/README.md index e02f652e..5318a712 100644 --- a/README.md +++ b/README.md @@ -401,6 +401,40 @@ This can be changed by creating a custom configuration file under [and finally](https://docs.docker.com/storage/bind-mounts/#choose-the--v-or---mount-flag) setting the Docker environment variable `SIR_CONFIG_PATH` to its path. +#### Customize backend Postgres server + +By default, the services `indexer` and `musicbrainz` are trying to connect to the host `db` (for both read-only and write host) but the hosts can +be customized using the `MUSICBRAINZ_POSTGRES_SERVER` and `MUSICBRAINZ_POSTGRES_READONLY_SERVER` environment variables. + +Notes: +* After switching to another Postgres server: + * If not transferring data, it is needed to create the database again. + * For live indexing, the RabbitMQ server has to still be reachable from the Postgres server. +* The helper scripts `check-search-indexes` and `create-amqp-extension` won’t work anymore. +* The service `db` will still be up even if unused. + +#### Customize backend RabbitMQ server + +By default, the services `db`, `indexer` and `musicbrainz` are trying to connect to the host `mq` +but the host can be customized using the `MUSICBRAINZ_RABBITMQ_SERVER` environment variable. + +Notes: +* After switching to another RabbitMQ server: + - Live indexing requires to go through AMQP Setup again. + - If not transferring data, it might be needed to build search indexes again. +* The helper script `purge-message-queues` won’t work anymore. +* The service `mq` will still be up even if unused. + +#### Customize backend Redis server + +By default, the service `musicbrainz` is trying to connect to the host `redis` +but the host can be customized using the `MUSICBRAINZ_REDIS_SERVER` environment variable. + +Notes: +* After switching to another Redis server: + - If not transferring data, MusicBrainz user sessions will be reset. +* The service `redis` will still be running even if unused. + ### Docker Compose overrides In Docker Compose, it is possible to override the base configuration using diff --git a/build/musicbrainz-dev/DBDefs.pm b/build/musicbrainz-dev/DBDefs.pm index 5d050b60..ffa5cafe 100644 --- a/build/musicbrainz-dev/DBDefs.pm +++ b/build/musicbrainz-dev/DBDefs.pm @@ -54,7 +54,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases( database => "musicbrainz_db", username => "$ENV{POSTGRES_USER}", password => "$ENV{POSTGRES_PASSWORD}", - host => "db", + host => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}", port => "5432", }, # How to connect to a test database @@ -62,7 +62,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases( database => "musicbrainz_test", username => "$ENV{POSTGRES_USER}", password => "$ENV{POSTGRES_PASSWORD}", - host => "db", + host => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}", port => "5432", }, # How to connect to a Selenium test database. This database is created @@ -73,7 +73,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases( schema => 'musicbrainz', username => "$ENV{POSTGRES_USER}", password => "$ENV{POSTGRES_PASSWORD}", - host => "db", + host => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}", port => "5432", }, # How to connect for read-only access. See "REPLICATION_TYPE" (below) @@ -81,7 +81,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases( database => "musicbrainz_db", username => "$ENV{POSTGRES_USER}", password => "$ENV{POSTGRES_PASSWORD}", - host => "db", + host => "$ENV{MUSICBRAINZ_POSTGRES_READONLY_SERVER}", port => "5432", }, # How to connect for read-only access to the production database in standby mode. @@ -101,7 +101,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases( database => "template1", username => "$ENV{POSTGRES_USER}", password => "$ENV{POSTGRES_PASSWORD}", - host => "db", + host => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}", port => "5432", }, # How to connect when running maintenance scripts located under admin/. @@ -111,7 +111,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases( # database => "musicbrainz_db", # username => "$ENV{POSTGRES_USER}", # password => "$ENV{POSTGRES_PASSWORD}", -# host => "db", +# host => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}", # port => "5432", # }, # Fill out only if RAWDATA lives on a different host from the READWRITE server. @@ -274,7 +274,7 @@ sub PLUGIN_CACHE_OPTIONS { my $self = shift; return { class => 'MusicBrainz::Server::CacheWrapper::Redis', - server => 'redis:6379', + server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379", namespace => $self->CACHE_NAMESPACE . 'Catalyst:', }; } @@ -289,7 +289,7 @@ sub CACHE_MANAGER_OPTIONS { external => { class => 'MusicBrainz::Server::CacheWrapper::Redis', options => { - server => 'redis:6379', + server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379", namespace => $self->CACHE_NAMESPACE, }, }, @@ -328,7 +328,7 @@ sub DATASTORE_REDIS_ARGS { return { database => 0, namespace => $self->CACHE_NAMESPACE, - server => 'redis:6379', + server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379", test_database => 1, }; } diff --git a/build/musicbrainz-dev/Dockerfile b/build/musicbrainz-dev/Dockerfile index e52f769f..e679a00f 100644 --- a/build/musicbrainz-dev/Dockerfile +++ b/build/musicbrainz-dev/Dockerfile @@ -86,6 +86,10 @@ ENV BASH_ENV=/noninteractive.bash_env \ MUSICBRAINZ_BASE_DOWNLOAD_URL=https://data.metabrainz.org/pub/musicbrainz \ MUSICBRAINZ_CATALYST_DEBUG=0 \ MUSICBRAINZ_DEVELOPMENT_SERVER=1 \ + MUSICBRAINZ_POSTGRES_SERVER=db \ + MUSICBRAINZ_POSTGRES_READONLY_SERVER=db \ + MUSICBRAINZ_RABBITMQ_SERVER=mq \ + MUSICBRAINZ_REDIS_SERVER=redis \ MUSICBRAINZ_SEARCH_SERVER=search:8983/solr \ MUSICBRAINZ_SERVER_PROCESSES=1 \ MUSICBRAINZ_STANDALONE_SERVER=1 \ diff --git a/build/musicbrainz-dev/scripts/createdb.sh b/build/musicbrainz-dev/scripts/createdb.sh index 237df91c..4fdaaf4a 100755 --- a/build/musicbrainz-dev/scripts/createdb.sh +++ b/build/musicbrainz-dev/scripts/createdb.sh @@ -94,7 +94,7 @@ for F in "${DUMP_FILES[@]}"; do done echo "found existing dumps" -dockerize -wait tcp://db:5432 -timeout 60s sleep 0 +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0 update-perl.sh diff --git a/build/musicbrainz-dev/scripts/indexer-triggers.sh b/build/musicbrainz-dev/scripts/indexer-triggers.sh index e4a17e30..eb821f27 100755 --- a/build/musicbrainz-dev/scripts/indexer-triggers.sh +++ b/build/musicbrainz-dev/scripts/indexer-triggers.sh @@ -12,7 +12,7 @@ fi INDEXER_SQL_DIR="$1" -dockerize -wait tcp://db:5432 -timeout 60s sleep 0 +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0 cd /musicbrainz-server diff --git a/build/musicbrainz-dev/scripts/recreatedb.sh b/build/musicbrainz-dev/scripts/recreatedb.sh index b325d420..86ac8548 100755 --- a/build/musicbrainz-dev/scripts/recreatedb.sh +++ b/build/musicbrainz-dev/scripts/recreatedb.sh @@ -2,5 +2,5 @@ set -e -dockerize -wait tcp://db:5432 -timeout 60s sleep 0 -psql postgres -U musicbrainz -h db -c "DROP DATABASE IF EXISTS musicbrainz_db;"; createdb.sh "$@" +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0 +psql postgres -U musicbrainz -h "${MUSICBRAINZ_POSTGRES_SERVER}" -c "DROP DATABASE IF EXISTS musicbrainz_db;"; createdb.sh "$@" diff --git a/build/musicbrainz-dev/scripts/replication.sh b/build/musicbrainz-dev/scripts/replication.sh index 8d98a28d..238db901 100755 --- a/build/musicbrainz-dev/scripts/replication.sh +++ b/build/musicbrainz-dev/scripts/replication.sh @@ -2,5 +2,5 @@ set -e -dockerize -wait tcp://db:5432 -timeout 60s sleep 0 +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0 exec /musicbrainz-server/admin/cron/mirror.sh diff --git a/build/musicbrainz-dev/scripts/update-javascript.sh b/build/musicbrainz-dev/scripts/update-javascript.sh index b4788fd8..aa7286cc 100755 --- a/build/musicbrainz-dev/scripts/update-javascript.sh +++ b/build/musicbrainz-dev/scripts/update-javascript.sh @@ -6,6 +6,6 @@ cd /musicbrainz-server yarn -dockerize -wait tcp://db:5432 -timeout 60s -wait tcp://mq:5672 -timeout 60s -wait tcp://redis:6379 -timeout 60s +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s -wait "tcp://${MUSICBRAINZ_RABBITMQ_SERVER}:5672" -timeout 60s -wait "tcp://${MUSICBRAINZ_REDIS_SERVER}:6379" -timeout 60s ./script/compile_resources.sh --watch client server tests &> /compile_resources.log & diff --git a/build/musicbrainz/DBDefs.pm b/build/musicbrainz/DBDefs.pm index 4b9d23e7..b7bb3f43 100644 --- a/build/musicbrainz/DBDefs.pm +++ b/build/musicbrainz/DBDefs.pm @@ -54,7 +54,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases( database => "musicbrainz_db", username => "$ENV{POSTGRES_USER}", password => "$ENV{POSTGRES_PASSWORD}", - host => "db", + host => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}", port => "5432", }, # How to connect to a test database @@ -80,7 +80,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases( database => "musicbrainz_db", username => "$ENV{POSTGRES_USER}", password => "$ENV{POSTGRES_PASSWORD}", - host => "db", + host => "$ENV{MUSICBRAINZ_POSTGRES_READONLY_SERVER}", port => "5432", }, # How to connect for administrative access @@ -88,7 +88,7 @@ MusicBrainz::Server::DatabaseConnectionFactory->register_databases( database => "template1", username => "$ENV{POSTGRES_USER}", password => "$ENV{POSTGRES_PASSWORD}", - host => "db", + host => "$ENV{MUSICBRAINZ_POSTGRES_SERVER}", port => "5432", }, # How to connect when running maintenance scripts located under admin/. @@ -261,7 +261,7 @@ sub PLUGIN_CACHE_OPTIONS { my $self = shift; return { class => 'MusicBrainz::Server::CacheWrapper::Redis', - server => 'redis:6379', + server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379", namespace => $self->CACHE_NAMESPACE . 'Catalyst:', }; } @@ -276,7 +276,7 @@ sub CACHE_MANAGER_OPTIONS { external => { class => 'MusicBrainz::Server::CacheWrapper::Redis', options => { - server => 'redis:6379', + server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379", namespace => $self->CACHE_NAMESPACE, }, }, @@ -315,7 +315,7 @@ sub DATASTORE_REDIS_ARGS { return { database => 0, namespace => $self->CACHE_NAMESPACE, - server => 'redis:6379', + server => "$ENV{MUSICBRAINZ_REDIS_SERVER}:6379", test_database => 1, }; } diff --git a/build/musicbrainz/Dockerfile b/build/musicbrainz/Dockerfile index e3e4e9c6..eb57eee8 100644 --- a/build/musicbrainz/Dockerfile +++ b/build/musicbrainz/Dockerfile @@ -84,6 +84,10 @@ ENV BASH_ENV=/noninteractive.bash_env \ MUSICBRAINZ_CATALYST_DEBUG=0 \ MUSICBRAINZ_DB_SCHEMA_SEQUENCE=28 \ MUSICBRAINZ_DEVELOPMENT_SERVER=0 \ + MUSICBRAINZ_POSTGRES_SERVER=db \ + MUSICBRAINZ_POSTGRES_READONLY_SERVER=db \ + MUSICBRAINZ_RABBITMQ_SERVER=mq \ + MUSICBRAINZ_REDIS_SERVER=redis \ MUSICBRAINZ_SEARCH_SERVER=search:8983/solr \ MUSICBRAINZ_SERVER_PROCESSES=10 \ MUSICBRAINZ_STANDALONE_SERVER=0 \ diff --git a/build/musicbrainz/scripts/createdb.sh b/build/musicbrainz/scripts/createdb.sh index 89f136ac..b25d3800 100755 --- a/build/musicbrainz/scripts/createdb.sh +++ b/build/musicbrainz/scripts/createdb.sh @@ -94,7 +94,7 @@ for F in "${DUMP_FILES[@]}"; do done echo "found existing dumps" -dockerize -wait tcp://db:5432 -timeout 60s sleep 0 +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0 mkdir -p $TMP_DIR cd /media/dbdump diff --git a/build/musicbrainz/scripts/indexer-triggers.sh b/build/musicbrainz/scripts/indexer-triggers.sh index 9c7099c2..79d7153b 100755 --- a/build/musicbrainz/scripts/indexer-triggers.sh +++ b/build/musicbrainz/scripts/indexer-triggers.sh @@ -12,7 +12,7 @@ fi INDEXER_SQL_DIR="$1" -dockerize -wait tcp://db:5432 -timeout 60s sleep 0 +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0 cd /musicbrainz-server diff --git a/build/musicbrainz/scripts/load-crontab-only.sh b/build/musicbrainz/scripts/load-crontab-only.sh index 8dcd6fd8..0935f3ee 100755 --- a/build/musicbrainz/scripts/load-crontab-only.sh +++ b/build/musicbrainz/scripts/load-crontab-only.sh @@ -3,8 +3,8 @@ set -e -u dockerize \ - -wait tcp://db:5432 -timeout 60s \ - -wait tcp://redis:6379 -timeout 60s \ + -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s \ + -wait "tcp://${MUSICBRAINZ_REDIS_SERVER}:6379" -timeout 60s \ true if [ -f /crons.conf -a -s /crons.conf ] diff --git a/build/musicbrainz/scripts/recreatedb.sh b/build/musicbrainz/scripts/recreatedb.sh index b325d420..86ac8548 100755 --- a/build/musicbrainz/scripts/recreatedb.sh +++ b/build/musicbrainz/scripts/recreatedb.sh @@ -2,5 +2,5 @@ set -e -dockerize -wait tcp://db:5432 -timeout 60s sleep 0 -psql postgres -U musicbrainz -h db -c "DROP DATABASE IF EXISTS musicbrainz_db;"; createdb.sh "$@" +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0 +psql postgres -U musicbrainz -h "${MUSICBRAINZ_POSTGRES_SERVER}" -c "DROP DATABASE IF EXISTS musicbrainz_db;"; createdb.sh "$@" diff --git a/build/musicbrainz/scripts/replication.sh b/build/musicbrainz/scripts/replication.sh index 8d98a28d..238db901 100755 --- a/build/musicbrainz/scripts/replication.sh +++ b/build/musicbrainz/scripts/replication.sh @@ -2,5 +2,5 @@ set -e -dockerize -wait tcp://db:5432 -timeout 60s sleep 0 +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s sleep 0 exec /musicbrainz-server/admin/cron/mirror.sh diff --git a/build/musicbrainz/scripts/start.sh b/build/musicbrainz/scripts/start.sh index 6e5b86e0..ef4892e6 100755 --- a/build/musicbrainz/scripts/start.sh +++ b/build/musicbrainz/scripts/start.sh @@ -9,7 +9,7 @@ then /musicbrainz-server/script/compile_resources.sh fi -dockerize -wait tcp://db:5432 -timeout 60s -wait tcp://mq:5672 -timeout 60s -wait tcp://redis:6379 -timeout 60s start_mb_renderer.pl +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s -wait "tcp://${MUSICBRAINZ_RABBITMQ_SERVER}:5672" -timeout 60s -wait "tcp://${MUSICBRAINZ_REDIS_SERVER}:6379" -timeout 60s start_mb_renderer.pl if [ -f /crons.conf -a -s /crons.conf ] then diff --git a/build/musicbrainz/scripts/upgrade-db-schema.sh b/build/musicbrainz/scripts/upgrade-db-schema.sh index c5da90ff..4ed5a580 100755 --- a/build/musicbrainz/scripts/upgrade-db-schema.sh +++ b/build/musicbrainz/scripts/upgrade-db-schema.sh @@ -4,4 +4,4 @@ set -e -u export MUSICBRAINZ_DB_SCHEMA_SEQUENCE=27 -dockerize -wait tcp://db:5432 -timeout 60s ./upgrade.sh +dockerize -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s ./upgrade.sh diff --git a/build/sir-dev/scripts/docker-entrypoint.sh b/build/sir-dev/scripts/docker-entrypoint.sh index a451e6d3..361d5308 100755 --- a/build/sir-dev/scripts/docker-entrypoint.sh +++ b/build/sir-dev/scripts/docker-entrypoint.sh @@ -12,8 +12,8 @@ pip install \ -r requirements_dev.txt dockerize \ - -wait tcp://db:5432 -timeout 60s \ - -wait tcp://mq:5672 -timeout 60s \ + -wait "tcp://${MUSICBRAINZ_POSTGRES_SERVER}:5432" -timeout 60s \ + -wait "tcp://${MUSICBRAINZ_RABBITMQ_SERVER}:5672" -timeout 60s \ -wait tcp://search:8983 -timeout 60s \ sleep 0 diff --git a/build/sir/Dockerfile b/build/sir/Dockerfile index 6666f350..a38c5492 100644 --- a/build/sir/Dockerfile +++ b/build/sir/Dockerfile @@ -46,5 +46,11 @@ RUN git clone --depth=1 --branch "v${SIR_VERSION}" https://github.com/metabrainz WORKDIR /code +ENV MUSICBRAINZ_RABBITMQ_SERVER=mq \ + MUSICBRAINZ_POSTGRES_SERVER=db \ + MUSICBRAINZ_POSTGRES_READONLY_SERVER=db \ + MUSICBRAINZ_SEARCH_SERVER=search:8983/solr + ENV POSTGRES_USER=musicbrainz ENV POSTGRES_PASSWORD=musicbrainz + diff --git a/default/indexer.ini b/default/indexer.ini index 6198c3fd..d421f495 100644 --- a/default/indexer.ini +++ b/default/indexer.ini @@ -1,12 +1,12 @@ [database] dbname = musicbrainz_db -host = db +host = ${MUSICBRAINZ_POSTGRES_SERVER} password = ${POSTGRES_PASSWORD} port = 5432 user = ${POSTGRES_USER} [solr] -uri = http://search:8983/solr +uri = ${MUSICBRAINZ_SEARCH_SERVER} batch_size = 200 [sir] @@ -18,7 +18,7 @@ query_batch_size = 5000 wscompat = on [rabbitmq] -host = mq +host = ${MUSICBRAINZ_RABBITMQ_SERVER} user = sir password = sir vhost = /search-index-rebuilder