Flask: add Prometheus endpoint (#983)

This is to help gain insights for #950.
Leverage [prometheus-flask-exporter](https://github.com/rycus86/prometheus_flask_exporter), a high-level client atop Prometheus' Python client designed for Flask and Gunicorn.
Use `/tmp` for the multiprocess directory to temporarily write metrics for the manager process, which is always writable and reset when the container is recreated, as required.
Add alternate gunicorn configuration for testing this metrics setup without going straight into production.

Integrate entrypoints into Dockerfile, now that stager-dev is no longer needed.
Disable pip cache when building images to reduce image size.

Dockerfile

@@ -11,11 +11,12 @@ ENV GIT_SHA=${GIT_SHA}
 WORKDIR /usr/src/stager
 # Install PyPI prod-only packages first and then copy the MinIO client as the latter updates more frequently
 COPY requirements.txt .
-RUN pip3 install -r requirements.txt
+RUN pip3 install --no-cache-dir -r requirements.txt
 COPY --from=mc /usr/bin/mc /usr/bin/mc
 COPY . .
 ENV FLASK_ENV production
-EXPOSE 5000
+ENV PROMETHEUS_MULTIPROC_DIR /tmp
+EXPOSE 5000 8080
 # Prevent accidentally using this image for development by adding the prod server arguments in the entrypoint
-ENTRYPOINT ["gunicorn", "wsgi:app", "--bind", "0.0.0.0:5000", "--access-logfile", "-", "--log-file", "-"]
-CMD ["--preload", "--workers", "2", "--threads", "2"]
+# Automatically run migrations on startup
+ENTRYPOINT ["./utils/run.sh", "prod", "--bind", "0.0.0.0:5000", "--access-logfile", "-", "--log-file", "-"]

docker-compose.ccm.yaml

@@ -8,8 +8,6 @@ x-common: &common
 x-app: &app
   image: "ghcr.io/ccmbioinfo/stager:${ST_VERSION}"
   user: www-data
-  # Not in Dockerfile yet because the base image is used on stager-dev
-  entrypoint: ./utils/run.sh prod --bind 0.0.0.0:5000 --access-logfile - --log-file -
   command: --preload --workers ${GUNICORN_WORKERS:-1}
   healthcheck:
     test: ["CMD", "./healthcheck.py"]

docker-compose.cheo.yaml

@@ -18,8 +18,7 @@ services:
       SENDGRID_FROM_EMAIL:
     ports:
       - "5000:5000"
-    # Not in Dockerfile yet because the base image is used on stager-dev
-    entrypoint: ./utils/run.sh prod --bind 0.0.0.0:5000 --access-logfile - --log-file -
+      - "9121:8080"
     command: --preload --workers ${GUNICORN_WORKERS:-1}
     healthcheck:
       test: ["CMD", "./healthcheck.py"]

docker-compose.yaml

@@ -58,6 +58,7 @@ services:
       MINIO_REGION_NAME:
       FLASK_DEBUG: 1
       PYTHONDONTWRITEBYTECODE: 1
+      DEBUG_METRICS: 1
     ports:
       - "${FLASK_HOST_PORT:-127.0.0.1:5000}:5000"
       - "${DEBUG_HOST_PORT:-127.0.0.1:5678}:5678"
@@ -67,3 +68,26 @@ services:
     volumes:
       - ./flask:/usr/src/stager
     entrypoint: ["python", "-m", "debugpy", "--listen", "0.0.0.0:5678", "-m", "flask", "run", "--host=0.0.0.0"]
+  # Simulated production gunicorn configuration
+  app_gunicorn:
+    build:
+      context: flask
+      dockerfile: ../Dockerfile
+      args:
+        GIT_SHA:
+    image: ghcr.io/ccmbioinfo/stager:latest
+    profiles:
+      - gunicorn
+    environment:
+      ST_SECRET_KEY:
+      ST_DATABASE_URI: "mysql+pymysql://${MYSQL_USER}:${MYSQL_PASSWORD}@mysql/${MYSQL_DATABASE}"
+      MINIO_ENDPOINT: minio:9000
+      MINIO_ACCESS_KEY:
+      MINIO_SECRET_KEY:
+      MINIO_REGION_NAME:
+    ports:
+      - "${FLASK_HOST_PORT:-127.0.0.1:5000}:5000"
+      - "${METRICS_HOST_PORT:-127.0.0.1:8080}:8080"
+    depends_on:
+      - mysql
+      - minio

flask/Dockerfile

@@ -8,7 +8,7 @@ LABEL org.opencontainers.image.vendor Centre for Computational Medicine
 WORKDIR /usr/src/stager
 # Install PyPI packages first and then copy the MinIO client as the latter updates more frequently
 COPY requirements-dev.txt .
-RUN pip3 install -r requirements-dev.txt
+RUN pip3 install --no-cache-dir -r requirements-dev.txt
 COPY --from=mc /usr/bin/mc /usr/bin/mc
 ENV FLASK_ENV development
 # Prevent accidentally running this image in production.

flask/app/__init__.py

@@ -23,7 +23,7 @@
 from flask import Flask
 from flask import logging as flask_logging
 
-from .extensions import db, login, ma, migrate, oauth
+from .extensions import db, login, ma, metrics, migrate, oauth
 from .tasks import send_email_notification
 from .utils import DateTimeEncoder
 
@@ -91,6 +91,8 @@ def register_extensions(app):
         server_metadata_url=app.config["OIDC_WELL_KNOWN"],
         client_kwargs={"scope": "openid"},
     )
+    metrics.init_app(app)
+    metrics.info("stager", "Stager process info", revision=app.config.get("GIT_SHA"))
 
 
 def config_logger(app):

flask/app/extensions.py

@@ -1,12 +1,22 @@
+from os import getenv
 from flask_login import LoginManager
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy
 from authlib.integrations.flask_client import OAuth
+from prometheus_flask_exporter import PrometheusMetrics
+from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics
 
 db = SQLAlchemy()
 login = LoginManager()
 migrate = Migrate(compare_type=True)
 oauth = OAuth()
 ma = Marshmallow()
 login.session_protection = "strong"
+
+if getenv("FLASK_ENV") == "development":
+    # Note that /metrics will not be live without DEBUG_METRICS set
+    metrics = PrometheusMetrics(None)
+else:  # production
+    # Must additionally set PROMETHEUS_MULTIPROC_DIR
+    metrics = GunicornPrometheusMetrics(None, path=None)

flask/gunicorn.conf.py

@@ -0,0 +1,9 @@
+from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics
+
+
+def when_ready(server):
+    GunicornPrometheusMetrics.start_http_server_when_ready(8080)
+
+
+def child_exit(server, worker):
+    GunicornPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid)

flask/requirements-dev.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.7
+# This file is autogenerated by pip-compile
 # To update, run:
 #
 #    pip-compile requirements-dev.in
@@ -14,6 +14,10 @@ attrs==20.3.0
     # via pytest
 authlib==0.15.5
     # via -r requirements.in
+backports.zoneinfo==0.2.1
+    # via
+    #   pytz-deprecation-shim
+    #   tzlocal
 black==21.11b1
     # via -r requirements-dev.in
 certifi==2020.12.5
@@ -34,13 +38,6 @@ cryptography==3.4.7
     #   pymysql
 debugpy==1.5.1
     # via -r requirements-dev.in
-flask==2.0.2
-    # via
-    #   -r requirements.in
-    #   flask-login
-    #   flask-marshmallow
-    #   flask-migrate
-    #   flask-sqlalchemy
 flask-login==0.5.0
     # via -r requirements.in
 flask-marshmallow==0.14.0
@@ -51,6 +48,14 @@ flask-sqlalchemy==2.5.1
     # via
     #   -r requirements.in
     #   flask-migrate
+flask==2.0.2
+    # via
+    #   -r requirements.in
+    #   flask-login
+    #   flask-marshmallow
+    #   flask-migrate
+    #   flask-sqlalchemy
+    #   prometheus-flask-exporter
 greenlet==1.1.2
     # via sqlalchemy
 gunicorn==20.1.0
@@ -73,12 +78,12 @@ markupsafe==2.0.1
     # via
     #   jinja2
     #   mako
+marshmallow-sqlalchemy==0.26.1
+    # via -r requirements.in
 marshmallow==3.14.1
     # via
     #   flask-marshmallow
     #   marshmallow-sqlalchemy
-marshmallow-sqlalchemy==0.26.1
-    # via -r requirements.in
 mccabe==0.6.1
     # via pylint
 minio==7.1.2
@@ -99,6 +104,10 @@ platformdirs==2.2.0
     #   pylint
 pluggy==0.13.1
     # via pytest
+prometheus-client==0.12.0
+    # via prometheus-flask-exporter
+prometheus-flask-exporter==0.18.6
+    # via -r requirements.in
 py==1.10.0
     # via pytest
 pycparser==2.20
@@ -119,12 +128,12 @@ python-editor==1.0.4
     # via alembic
 python-http-client==3.3.3
     # via sendgrid
+pytz-deprecation-shim==0.1.0.post0
+    # via tzlocal
 pytz==2021.1
     # via
     #   apscheduler
     #   pandas
-pytz-deprecation-shim==0.1.0.post0
-    # via tzlocal
 regex==2021.4.4
     # via black
 requests==2.26.0
@@ -151,7 +160,10 @@ toml==0.10.2
 tomli==1.1.0
     # via black
 typing-extensions==3.10.0.2
-    # via black
+    # via
+    #   astroid
+    #   black
+    #   pylint
 tzdata==2021.5
     # via pytz-deprecation-shim
 tzlocal==4.1

flask/requirements.in

@@ -1,15 +1,16 @@
 apscheduler
 authlib
 flask
-flask_sqlalchemy
 flask_login
+flask-marshmallow
 flask_migrate
+flask_sqlalchemy
 gunicorn
+marshmallow-sqlalchemy
 minio
 pandas
+prometheus-flask-exporter
 pymysql[rsa]
 requests
+sendgrid
 sqlalchemy
-flask-marshmallow
-marshmallow-sqlalchemy
-sendgrid

flask/requirements.txt

@@ -1,15 +1,19 @@
 #
-# This file is autogenerated by pip-compile with python 3.7
+# This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile requirements.in
+#    pip-compile
 #
 alembic==1.5.8
     # via flask-migrate
 apscheduler==3.8.1
     # via -r requirements.in
 authlib==0.15.5
     # via -r requirements.in
+backports.zoneinfo==0.2.1
+    # via
+    #   pytz-deprecation-shim
+    #   tzlocal
 certifi==2020.12.5
     # via
     #   minio
@@ -24,13 +28,6 @@ cryptography==3.4.7
     # via
     #   authlib
     #   pymysql
-flask==2.0.2
-    # via
-    #   -r requirements.in
-    #   flask-login
-    #   flask-marshmallow
-    #   flask-migrate
-    #   flask-sqlalchemy
 flask-login==0.5.0
     # via -r requirements.in
 flask-marshmallow==0.14.0
@@ -41,6 +38,14 @@ flask-sqlalchemy==2.5.1
     # via
     #   -r requirements.in
     #   flask-migrate
+flask==2.0.2
+    # via
+    #   -r requirements.in
+    #   flask-login
+    #   flask-marshmallow
+    #   flask-migrate
+    #   flask-sqlalchemy
+    #   prometheus-flask-exporter
 greenlet==1.1.2
     # via sqlalchemy
 gunicorn==20.1.0
@@ -57,18 +62,22 @@ markupsafe==2.0.1
     # via
     #   jinja2
     #   mako
+marshmallow-sqlalchemy==0.26.1
+    # via -r requirements.in
 marshmallow==3.14.1
     # via
     #   flask-marshmallow
     #   marshmallow-sqlalchemy
-marshmallow-sqlalchemy==0.26.1
-    # via -r requirements.in
 minio==7.1.2
     # via -r requirements.in
 numpy==1.21.4
     # via pandas
 pandas==1.3.4
     # via -r requirements.in
+prometheus-client==0.12.0
+    # via prometheus-flask-exporter
+prometheus-flask-exporter==0.18.6
+    # via -r requirements.in
 pycparser==2.20
     # via cffi
 pymysql[rsa]==1.0.2
@@ -81,12 +90,12 @@ python-editor==1.0.4
     # via alembic
 python-http-client==3.3.3
     # via sendgrid
+pytz-deprecation-shim==0.1.0.post0
+    # via tzlocal
 pytz==2021.1
     # via
     #   apscheduler
     #   pandas
-pytz-deprecation-shim==0.1.0.post0
-    # via tzlocal
 requests==2.26.0
     # via -r requirements.in
 sendgrid==6.9.2