Merge pull request #64 from bird-house/true-plugable-components

Improved plugable component architecture.

Before this PR, components needing default values, needing template variable substitution, needing to execute commands pre and post `docker-compose up` are hardcoding their needs directly to the "core" system, basically "leaking" their requirements out even when they are not activated (fixes #62).

This PR provides true plugable architecture for the components so they can provide all their needs without having to modify the code of the "core" system.

All the components (monitoring, generic_bird, emu, testthredds) are modified to leverage the new plugable architecture, with additional customizations given it is cleaner/easier to have default configuration values.

Given this PR both changes the architecture and modify many components at the same time, it is best to read each commit separately to easier understand which code change belongs to which "goal".

Deployed here https://lvupavicsmaster.ouranos.ca with all the impacted components activated to test the change:
* Canarie: https://lvupavicsmaster.ouranos.ca/canarie/node/service/status
* Generic bird (using Finch): https://lvupavicsmaster.ouranos.ca/twitcher/ows/proxy/generic_bird?service=WPS&version=1.0.0&request=GetCapabilities
* Emu: https://lvupavicsmaster.ouranos.ca/twitcher/ows/proxy/emu?service=WPS&version=1.0.0&request=GetCapabilities
* Test Thredds: https://lvupavicsmaster.ouranos.ca/testthredds/catalog.html
* Prometheus: http://lvupavicsmaster.ouranos.ca:9090/alerts
* AlertManager: http://lvupavicsmaster.ouranos.ca:9093/
* Grafana dashboard: http://lvupavicsmaster.ouranos.ca:3001/d/pf6xQMWGz/docker-and-system-monitoring?orgId=1&refresh=5m

README.md

@@ -6,5 +6,6 @@ for a full-fledged production platform.
 Power Analytics and Visualization for Climate Science - Powered by Birdhouse and other ESGF software
 
 How to deploy the entire PAVICS platform, see the
-[README](birdhouse/README.md) and the
+[README](birdhouse/README.md) and the various extra components 
+[README for extra core components](birdhouse/components/README.rst),
 [README for optional components](birdhouse/optional-components/README.md).

birdhouse/components/monitoring/default.env

@@ -0,0 +1,28 @@
+# All env in this default.env can be overridden by env.local.
+# All env in this default.env must not depend on any env in env.local.
+
+export GRAFANA_ADMIN_PASSWORD="changeme!"
+export ALERTMANAGER_ADMIN_EMAIL_RECEIVER=""  # "user1@example.com,user2@example.com"
+export SMTP_SERVER=""  # "smtp.example.com:25"
+export ALERTMANAGER_EXTRA_GLOBAL=""
+export ALERTMANAGER_EXTRA_ROUTES=""
+export ALERTMANAGER_EXTRA_INHIBITION=""
+export ALERTMANAGER_EXTRA_RECEIVERS=""
+
+
+
+# add vars only needed to be substituted in templates
+
+VARS="
+  $VARS
+  \$ALERTMANAGER_ADMIN_EMAIL_RECEIVER
+  \$SMTP_SERVER
+"
+
+OPTIONAL_VARS="
+  $OPTIONAL_VARS
+  \$ALERTMANAGER_EXTRA_GLOBAL
+  \$ALERTMANAGER_EXTRA_ROUTES
+  \$ALERTMANAGER_EXTRA_INHIBITION
+  \$ALERTMANAGER_EXTRA_RECEIVERS
+"

birdhouse/components/monitoring/pre-docker-compose-up

@@ -0,0 +1,5 @@
+#!/bin/sh -x
+
+docker volume create prometheus_persistence  # metrics db
+docker volume create grafana_persistence  # dashboard and config db
+docker volume create alertmanager_persistence  # storage

birdhouse/config/thredds/entrypointwrapper

@@ -20,8 +20,12 @@ fi
 # https://www.unidata.ucar.edu/software/tds/v4.6/reference/TomcatBehindProxyServer.html
 WEBAPPS_ROOT="/usr/local/tomcat/webapps"
 EXISTING_CONTEXT_ROOT="thredds"
-WANTED_CONTEXT_ROOT="twitcher/ows/proxy/thredds"
-WANTED_CONTEXT_ROOT_WARFILE_NAME="twitcher#ows#proxy#thredds"
+if [ -z "$WANTED_CONTEXT_ROOT" ]; then
+    WANTED_CONTEXT_ROOT="twitcher/ows/proxy/thredds"
+fi
+if [ -z "$WANTED_CONTEXT_ROOT_WARFILE_NAME" ]; then
+    WANTED_CONTEXT_ROOT_WARFILE_NAME="twitcher#ows#proxy#thredds"
+fi
 
 if [ -d "$WEBAPPS_ROOT/$EXISTING_CONTEXT_ROOT" ]; then
     mv "$WEBAPPS_ROOT/$EXISTING_CONTEXT_ROOT" "$WEBAPPS_ROOT/$WANTED_CONTEXT_ROOT_WARFILE_NAME"

birdhouse/common.env → birdhouse/default.env

@@ -1,12 +1,12 @@
-# All env this common.env can be overridden by env.local.
+# All env in this default.env can be overridden by env.local.
+# All env in this default.env must not depend on any env in env.local.
 
 # Jupyter single-user server image
 export DOCKER_NOTEBOOK_IMAGE="pavics/workflow-tests:200803"
 
 export FINCH_IMAGE="birdhouse/finch:version-0.5.2"
-export GENERIC_BIRD_IMAGE="$FINCH_IMAGE"
-export GENERIC_BIRD_PORT="8010"
-export GENERIC_BIRD_NAME="generic_bird"
+
+export THREDDS_IMAGE="unidata/thredds-docker:4.6.14"
 
 # Folder on the host to persist Jupyter user data (noteboooks, HOME settings)
 export JUPYTERHUB_USER_DATA_DIR="/data/jupyterhub_user_data"
@@ -25,6 +25,12 @@ export JUPYTER_LOGIN_BANNER_BOTTOM_SECTION=""
 # Folder inside "proxy" container to drop extra monitoring config
 export CANARIE_MONITORING_EXTRA_CONF_DIR="/conf.d"
 
+export THREDDS_ORGANIZATION="Birdhouse"
+
+export MAGPIE_DB_NAME="magpiedb"
+
+export VERIFY_SSL="true"
+
 # Folder containing ssh deploy keys for all extra git repos
 #
 # Note when overriding this variable in env.local, do not use HOME environment

birdhouse/deployment/certbotwrapper

@@ -18,7 +18,7 @@ THIS_FILE="`realpath "$0"`"
 THIS_DIR="`dirname "$THIS_FILE"`"
 
 # Default values
-. $THIS_DIR/../common.env
+. $THIS_DIR/../default.env
 
 if [ -e "$THIS_DIR/../env.local" ]; then
     # Override default values

birdhouse/deployment/deploy.sh

@@ -121,10 +121,10 @@ done
 
 cd $COMPOSE_DIR
 
-. ./common.env
+. ./default.env
 
 set +x  # hide passwd in env.local in logs
-# reload again after common.env since env.local can override common.env
+# reload again after default.env since env.local can override default.env
 . $ENV_LOCAL_FILE
 set -x
 
@@ -162,10 +162,10 @@ done
 cd $COMPOSE_DIR
 
 # reload again after git pull because this file could be changed by the pull
-. ./common.env
+. ./default.env
 
 set +x  # hide passwd in env.local in logs
-# reload again after common.env since env.local can override common.env
+# reload again after default.env since env.local can override default.env
 # (ex: JUPYTERHUB_USER_DATA_DIR)
 . $ENV_LOCAL_FILE
 set -x

birdhouse/deployment/fix-write-perm

@@ -39,7 +39,7 @@ THIS_DIR="`dirname "$THIS_FILE"`"
 cd $THIS_DIR/../..
 
 # Default values
-. birdhouse/common.env
+. birdhouse/default.env
 
 if [ -e "birdhouse/env.local" ]; then
     # Override default values

birdhouse/deployment/install-deploy-notebook

@@ -29,7 +29,7 @@ if [ ! -e "$REPO_ROOT/birdhouse/deployment/trigger-deploy-notebook" ]; then
     exit 2
 fi
 
-. "$REPO_ROOT/birdhouse/common.env"
+. "$REPO_ROOT/birdhouse/default.env"
 
 if [ -f "$REPO_ROOT/birdhouse/env.local" ]; then
     # allow override of JUPYTERHUB_USER_DATA_DIR

birdhouse/deployment/trigger-deploy-notebook

@@ -15,7 +15,7 @@
 if [ -z "$JUPYTERHUB_USER_DATA_DIR" ]; then
   # running script manually (not with cron) source env.local file.
   COMPOSE_DIR="$(dirname -- "$(dirname -- "$(realpath "$0")")")"
-  . "$COMPOSE_DIR/common.env"  # default JUPYTERHUB_USER_DATA_DIR
+  . "$COMPOSE_DIR/default.env"  # default JUPYTERHUB_USER_DATA_DIR
   if [ -e "$COMPOSE_DIR/env.local" ]; then
       . "$COMPOSE_DIR/env.local"  # optional override JUPYTERHUB_USER_DATA_DIR
   fi

birdhouse/docker-compose.yml

@@ -228,7 +228,7 @@ services:
     restart: always
 
   thredds:
-    image: unidata/thredds-docker:4.6.14
+    image: ${THREDDS_IMAGE}
     container_name: thredds
     ports:
       - "8083:8080"

birdhouse/env.local.example

@@ -31,10 +31,11 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty
 # Optional vars
 #############################################################################
 
-# Extra dirs containing docker-compose-extra.yml file to override the default
-# docker-compose.yml file.  Inside each extra dir, there must be a
-# docker-compose-extra.yml file.  If this file is not found, the dir is
-# ignored.
+# Extra dirs possibly containing:
+# * `docker-compose-extra.yml` file to override the default docker-compose.yml file
+# * `default.env` file to provide extra defaults for each component
+# * `pre-docker-compose-up` script to execute before `docker-compose up`
+# * `post-docker-compose-up` script to execute after `docker-compose up`
 #
 # Useful to split configs into different dirs leveraging docker-compose
 # override capabilities, see https://docs.docker.com/compose/extends/.
@@ -51,9 +52,19 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty
 # * Assemble different combinations of components/functionalities by including
 #   only the config/docker-compose fragment necessary.
 #
-# Last dir/docker-compose fragment in the list have highest override precedence.
-# Ex: last docker-compose volume mount to same destination win over the
-# previous docker-compose volume mount.
+# Last dir/component in the EXTRA_CONF_DIRS list have highest override
+# precedence, example:
+#
+#   * Last docker-compose volume mount to same destination win over the
+#     previous docker-compose volume mount.
+#
+#   * Last default.env can change the values of all previous default.env.
+#
+#   * Last pre/post docker-compose-up script can potentially undo actions
+#     from previous scripts.
+#
+# Suggested to keep the private-config-repo last in the list to be able to
+# override anything.
 #
 # Format: space separated list of dirs
 #
@@ -107,7 +118,7 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty
 #export AUTODEPLOY_DEPLOY_KEY_ROOT_DIR="/path/to/ssh-deploy-keys-for-all-repos"
 
 # Frequency to trigger the various autodeploy tasks.
-# See common.env for default.
+# See default.env for default.
 #
 # For all possible syntax, see implementation at
 # https://github.com/Ouranosinc/docker-crontab/blob/3ac8cfa363b3f2ffdd0ead6089d355ff84521dc9/docker-entrypoint#L137-L184
@@ -207,7 +218,7 @@ export POSTGRES_MAGPIE_PASSWORD=postgres-qwerty
 #############################################################################
 #
 # Below are Mandatory if monitoring component is enabled:
-export GRAFANA_ADMIN_PASSWORD=changeme!
+#export GRAFANA_ADMIN_PASSWORD=changeme!
 #export ALERTMANAGER_ADMIN_EMAIL_RECEIVER="user1@example.com,user2@example.com"
 #export SMTP_SERVER="smtp.example.com:25"
 

birdhouse/optional-components/README.md

@@ -17,15 +17,32 @@ How to enable this config in `env.local` (a copy from
 
 ## Emu WPS service for testing
 
+Preconfigured for Emu but can also be used to quickly deploy any birds
+temporarily without changing code.  Good to preview new birds or test
+alternative configuration of existing birds.
+
+No Postgres DB configured.  If need Postgres DB, use generic_bird component
+instead.
+
 How to enable Emu in `env.local` (a copy from
 [`env.local.example`](../env.local.example)):
 
 * Add `./optional-components/emu` to `EXTRA_CONF_DIRS`.
-* Set `EMU_IMAGE`.
+* Optionally set `EMU_IMAGE`, `EMU_PORT`,
+  `EMU_NAME`, `EMU_INTERNAL_PORT`,
+  `EMU_WPS_OUTPUTS_VOL` in `env.local` for further customizations.
+  Default values are in [`emu/default.env`](emu/default.env).
+
+Emu service will be available at `http://PAVICS_FQDN:EMU_PORT/wps` or
+`https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/EMU_NAME` where
+`PAVICS_FQDN`, `PAVICS_FQDN_PUBLIC` and `TWITCHER_PROTECTED_PATH` are defined
+in your `env.local`.
 
-Emu service will be available at `http://PAVICS_FQDN:8888/wps` or
-`https://PAVICS_FQDN_PUBLIC/twitcher/ows/proxy/emu` where `PAVICS_FQDN`
-and `PAVICS_FQDN_PUBLIC` are defined in your `env.local`.
+Magpie will be automatically configured to give complete public anonymous
+access for this Emu WPS service.
+
+Canarie monitoring will also be automatically configured for this Emu WPS
+service.
 
 
 ## A second Thredds server for testing
@@ -35,10 +52,19 @@ How to enable in `env.local` (a copy from
 
 * Add `./optional-components/testthredds` to `EXTRA_CONF_DIRS`.
 
-Test Thredds service will be available at `http://PAVICS_FQDN:8084/testthredds`
-or `https://PAVICS_FQDN_PUBLIC/testthredds` where `PAVICS_FQDN` and
+* Optionally set `TESTTHREDDS_IMAGE`, `TESTTHREDDS_PORT`,
+  `TESTTHREDDS_CONTEXT_ROOT`, `TESTTHREDDS_WARFILE_NAME`,
+  `TESTTHREDDS_INTERNAL_PORT`, `TESTTHREDDS_NAME`,  in `env.local` for further
+  customizations.  Default values are in
+  [`testthredds/default.env`](testthredds/default.env).
+
+Test Thredds service will be available at
+`http://PAVICS_FQDN:TESTTHREDDS_PORT/TESTTHREDDS_CONTEXT_ROOT` or
+`https://PAVICS_FQDN_PUBLIC/TESTTHREDDS_CONTEXT_ROOT` where `PAVICS_FQDN` and
 `PAVICS_FQDN_PUBLIC` are defined in your `env.local`.
 
+Use same docker image as regular Thredds by default but can be customized.
+
 New container have new `TestDatasets` with volume-mount to `/data/testdatasets`
 on the host.  So your testing `.nc` and `.ncml` files should be added to
 `/data/testdatasets` on the host for them to show up on this Test Thredds
@@ -52,6 +78,9 @@ server.
 No Twitcher/Magpie access control, this Test Thredds is directly behind the
 Nginx proxy.
 
+Canarie monitoring will also be automatically configured for this second
+Thredds server.
+
 
 ## A generic bird WPS service
 
@@ -63,8 +92,10 @@ How to enable in `env.local` (a copy from
 
 * Add `./optional-components/generic_bird` to `EXTRA_CONF_DIRS`.
 
-* Optionally set `GENERIC_BIRD_IMAGE`, `GENERIC_BIRD_PORT`, `GENERIC_BIRD_NAME` in `env.local`
-  for further customizations.  Default values are in [`common.env`](../common.env).
+* Optionally set `GENERIC_BIRD_IMAGE`, `GENERIC_BIRD_PORT`,
+  `GENERIC_BIRD_NAME`, `GENERIC_BIRD_INTERNAL_PORT`,
+  `GENERIC_BIRD_POSTGRES_IMAGE` in `env.local` for further customizations.
+  Default values are in [`generic_bird/default.env`](generic_bird/default.env).
 
 The WPS service will be available at `http://PAVICS_FQDN:GENERIC_BIRD_PORT/wps`
 or `https://PAVICS_FQDN_PUBLIC/TWITCHER_PROTECTED_PATH/GENERIC_BIRD_NAME` where

birdhouse/optional-components/emu/default.env

@@ -0,0 +1,23 @@
+# All env in this default.env can be overridden by env.local.
+# All env in this default.env must not depend on any env in env.local.
+
+# Should have been "birdhouse/emu" but at the moment the current config only
+# works with the "watchdog/jobqueue" branch so have to default to an image that
+# works by default.
+export EMU_IMAGE="tlvu/emu:watchdog"
+export EMU_PORT="8888"
+export EMU_INTERNAL_PORT="5000"
+# name in Twitcher/Magpie and Canarie monitoring
+export EMU_NAME="emu"
+# set to 'testwps_outputs' when used together with testthredds component
+export EMU_WPS_OUTPUTS_VOL="wps_outputs"
+
+
+
+# add vars only needed to be substituted in templates
+
+OPTIONAL_VARS="
+  $OPTIONAL_VARS
+  \$EMU_PORT
+  \$EMU_NAME
+"

birdhouse/optional-components/emu/docker-compose-extra.yml

@@ -6,10 +6,10 @@ services:
     environment:
       - PYWPS_CFG=/wps.cfg
     ports:
-      - "8888:5000"
+      - "${EMU_PORT}:${EMU_INTERNAL_PORT}"
     volumes:
       - ./optional-components/emu/wps.cfg:/wps.cfg
-      - ${EMU_WPS_OUTPUTS_VOL:-wps_outputs}:/data/wpsoutputs
+      - ${EMU_WPS_OUTPUTS_VOL}:/data/wpsoutputs
       - /tmp
     restart: always
 

birdhouse/optional-components/emu/emu-magpie-permission.cfg.template

@@ -1,15 +1,15 @@
 permissions:
-  - service: emu
+  - service: ${EMU_NAME}
     permission: getcapabilities
     group: anonymous
     action: create
 
-  - service: emu
+  - service: ${EMU_NAME}
     permission: describeprocess
     group: anonymous
     action: create
 
-  - service: emu
+  - service: ${EMU_NAME}
     permission: execute
     group: anonymous
     action: create

birdhouse/optional-components/emu/emu-magpie-provider.cfg.template

@@ -1,7 +1,7 @@
 providers:
-  emu:
-    url: http://${PAVICS_FQDN}:8888/wps
-    title: Emu
+  ${EMU_NAME}:
+    url: http://${PAVICS_FQDN}:${EMU_PORT}/wps
+    title: ${EMU_NAME}
     public: true
     c4i: false
     type: wps

birdhouse/optional-components/emu/emu_canarie_api_monitoring.py.template

@@ -1,12 +1,12 @@
 SERVICES['node']['monitoring'].update({
-    'Emu-public': {
+    '${EMU_NAME}-public': {
         'request': {
-            'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/emu?service=WPS&version=1.0.0&request=GetCapabilities'
+            'url': 'https://${PAVICS_FQDN_PUBLIC}/${TWITCHER_PROTECTED_PATH}/${EMU_NAME}?service=WPS&version=1.0.0&request=GetCapabilities'
         },
     },
-    'Emu': {
+    '${EMU_NAME}': {
         'request': {
-            'url': 'http://${PAVICS_FQDN}:8888/wps?service=WPS&version=1.0.0&request=GetCapabilities'
+            'url': 'http://${PAVICS_FQDN}:${EMU_PORT}/wps?service=WPS&version=1.0.0&request=GetCapabilities'
         }
     },
 })