Flask: add Prometheus endpoint

Dockerfile

@@ -11,11 +11,12 @@ ENV GIT_SHA=${GIT_SHA}
 WORKDIR /usr/src/stager
 # Install PyPI prod-only packages first and then copy the MinIO client as the latter updates more frequently
 COPY requirements.txt .
-RUN pip3 install -r requirements.txt
+RUN pip3 install --no-cache-dir -r requirements.txt
 COPY --from=mc /usr/bin/mc /usr/bin/mc
 COPY . .
 ENV FLASK_ENV production
-EXPOSE 5000
+ENV PROMETHEUS_MULTIPROC_DIR /tmp
+EXPOSE 5000 8080
 # Prevent accidentally using this image for development by adding the prod server arguments in the entrypoint
-ENTRYPOINT ["gunicorn", "wsgi:app", "--bind", "0.0.0.0:5000", "--access-logfile", "-", "--log-file", "-"]
-CMD ["--preload", "--workers", "2", "--threads", "2"]
+# Automatically run migrations on startup
+ENTRYPOINT ["./utils/run.sh", "prod", "--bind", "0.0.0.0:5000", "--access-logfile", "-", "--log-file", "-"]

docker-compose.ccm.yaml

@@ -8,8 +8,6 @@ x-common: &common
 x-app: &app
   image: "ghcr.io/ccmbioinfo/stager:${ST_VERSION}"
   user: www-data
-  # Not in Dockerfile yet because the base image is used on stager-dev
-  entrypoint: ./utils/run.sh prod --bind 0.0.0.0:5000 --access-logfile - --log-file -
   command: --preload --workers ${GUNICORN_WORKERS:-1}
   healthcheck:
     test: ["CMD", "./healthcheck.py"]

docker-compose.cheo.yaml

@@ -18,8 +18,7 @@ services:
       SENDGRID_FROM_EMAIL:
     ports:
       - "5000:5000"
-    # Not in Dockerfile yet because the base image is used on stager-dev
-    entrypoint: ./utils/run.sh prod --bind 0.0.0.0:5000 --access-logfile - --log-file -
+      - "9121:8080"
     command: --preload --workers ${GUNICORN_WORKERS:-1}
     healthcheck:
       test: ["CMD", "./healthcheck.py"]

docker-compose.yaml

@@ -58,6 +58,7 @@ services:
       MINIO_REGION_NAME:
       FLASK_DEBUG: 1
       PYTHONDONTWRITEBYTECODE: 1
+      DEBUG_METRICS: 1
     ports:
       - "${FLASK_HOST_PORT:-127.0.0.1:5000}:5000"
       - "${DEBUG_HOST_PORT:-127.0.0.1:5678}:5678"
@@ -67,3 +68,26 @@ services:
     volumes:
       - ./flask:/usr/src/stager
     entrypoint: ["python", "-m", "debugpy", "--listen", "0.0.0.0:5678", "-m", "flask", "run", "--host=0.0.0.0"]
+  # Simulated production gunicorn configuration
+  app_gunicorn:
+    build:
+      context: flask
+      dockerfile: ../Dockerfile
+      args:
+        GIT_SHA:
+    image: ghcr.io/ccmbioinfo/stager:latest
+    profiles:
+      - gunicorn
+    environment:
+      ST_SECRET_KEY:
+      ST_DATABASE_URI: "mysql+pymysql://${MYSQL_USER}:${MYSQL_PASSWORD}@mysql/${MYSQL_DATABASE}"
+      MINIO_ENDPOINT: minio:9000
+      MINIO_ACCESS_KEY:
+      MINIO_SECRET_KEY:
+      MINIO_REGION_NAME:
+    ports:
+      - "${FLASK_HOST_PORT:-127.0.0.1:5000}:5000"
+      - "${METRICS_HOST_PORT:-127.0.0.1:8080}:8080"
+    depends_on:
+      - mysql
+      - minio

flask/Dockerfile

@@ -8,7 +8,7 @@ LABEL org.opencontainers.image.vendor Centre for Computational Medicine
 WORKDIR /usr/src/stager
 # Install PyPI packages first and then copy the MinIO client as the latter updates more frequently
 COPY requirements-dev.txt .
-RUN pip3 install -r requirements-dev.txt
+RUN pip3 install --no-cache-dir -r requirements-dev.txt
 COPY --from=mc /usr/bin/mc /usr/bin/mc
 ENV FLASK_ENV development
 # Prevent accidentally running this image in production.

flask/app/__init__.py

@@ -23,7 +23,7 @@
 from flask import Flask
 from flask import logging as flask_logging
 
-from .extensions import db, login, ma, migrate, oauth
+from .extensions import db, login, ma, metrics, migrate, oauth
 from .tasks import send_email_notification
 from .utils import DateTimeEncoder
 
@@ -91,6 +91,8 @@ def register_extensions(app):
         server_metadata_url=app.config["OIDC_WELL_KNOWN"],
         client_kwargs={"scope": "openid"},
     )
+    metrics.init_app(app)
+    metrics.info("stager", "Stager process info", revision=app.config.get("GIT_SHA"))
 
 
 def config_logger(app):

flask/app/extensions.py

@@ -1,12 +1,22 @@
+from os import getenv
 from flask_login import LoginManager
 from flask_marshmallow import Marshmallow
 from flask_migrate import Migrate
 from flask_sqlalchemy import SQLAlchemy
 from authlib.integrations.flask_client import OAuth
+from prometheus_flask_exporter import PrometheusMetrics
+from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics
 
 db = SQLAlchemy()
 login = LoginManager()
 migrate = Migrate(compare_type=True)
 oauth = OAuth()
 ma = Marshmallow()
 login.session_protection = "strong"
+
+if getenv("FLASK_ENV") == "development":
+    # Note that /metrics will not be live without DEBUG_METRICS set
+    metrics = PrometheusMetrics(None)
+else:  # production
+    # Must additionally set PROMETHEUS_MULTIPROC_DIR
+    metrics = GunicornPrometheusMetrics(None, path=None)

flask/gunicorn.conf.py

@@ -0,0 +1,9 @@
+from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics
+
+
+def when_ready(server):
+    GunicornPrometheusMetrics.start_http_server_when_ready(8080)
+
+
+def child_exit(server, worker):
+    GunicornPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid)

flask/requirements-dev.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.7
+# This file is autogenerated by pip-compile
 # To update, run:
 #
 #    pip-compile requirements-dev.in
@@ -14,6 +14,10 @@ attrs==20.3.0
     # via pytest
 authlib==0.15.5
     # via -r requirements.in
+backports.zoneinfo==0.2.1
+    # via
+    #   pytz-deprecation-shim
+    #   tzlocal
 black==21.11b1
     # via -r requirements-dev.in
 certifi==2020.12.5
@@ -34,13 +38,6 @@ cryptography==3.4.7
     #   pymysql
 debugpy==1.5.1
     # via -r requirements-dev.in
-flask==2.0.2
-    # via
-    #   -r requirements.in
-    #   flask-login
-    #   flask-marshmallow
-    #   flask-migrate
-    #   flask-sqlalchemy
 flask-login==0.5.0
     # via -r requirements.in
 flask-marshmallow==0.14.0
@@ -51,6 +48,14 @@ flask-sqlalchemy==2.5.1
     # via
     #   -r requirements.in
     #   flask-migrate
+flask==2.0.2
+    # via
+    #   -r requirements.in
+    #   flask-login
+    #   flask-marshmallow
+    #   flask-migrate
+    #   flask-sqlalchemy
+    #   prometheus-flask-exporter
 greenlet==1.1.2
     # via sqlalchemy
 gunicorn==20.1.0
@@ -73,12 +78,12 @@ markupsafe==2.0.1
     # via
     #   jinja2
     #   mako
+marshmallow-sqlalchemy==0.26.1
+    # via -r requirements.in
 marshmallow==3.14.1
     # via
     #   flask-marshmallow
     #   marshmallow-sqlalchemy
-marshmallow-sqlalchemy==0.26.1
-    # via -r requirements.in
 mccabe==0.6.1
     # via pylint
 minio==7.1.2
@@ -99,6 +104,10 @@ platformdirs==2.2.0
     #   pylint
 pluggy==0.13.1
     # via pytest
+prometheus-client==0.12.0
+    # via prometheus-flask-exporter
+prometheus-flask-exporter==0.18.6
+    # via -r requirements.in
 py==1.10.0
     # via pytest
 pycparser==2.20
@@ -119,12 +128,12 @@ python-editor==1.0.4
     # via alembic
 python-http-client==3.3.3
     # via sendgrid
+pytz-deprecation-shim==0.1.0.post0
+    # via tzlocal
 pytz==2021.1
     # via
     #   apscheduler
     #   pandas
-pytz-deprecation-shim==0.1.0.post0
-    # via tzlocal
 regex==2021.4.4
     # via black
 requests==2.26.0
@@ -151,7 +160,10 @@ toml==0.10.2
 tomli==1.1.0
     # via black
 typing-extensions==3.10.0.2
-    # via black
+    # via
+    #   astroid
+    #   black
+    #   pylint
 tzdata==2021.5
     # via pytz-deprecation-shim
 tzlocal==4.1

flask/requirements.in

@@ -1,15 +1,16 @@
 apscheduler
 authlib
 flask
-flask_sqlalchemy
 flask_login
+flask-marshmallow
 flask_migrate
+flask_sqlalchemy
 gunicorn
+marshmallow-sqlalchemy
 minio
 pandas
+prometheus-flask-exporter
 pymysql[rsa]
 requests
+sendgrid
 sqlalchemy
-flask-marshmallow
-marshmallow-sqlalchemy
-sendgrid

flask/requirements.txt

@@ -1,15 +1,19 @@
 #
-# This file is autogenerated by pip-compile with python 3.7
+# This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile requirements.in
+#    pip-compile
 #
 alembic==1.5.8
     # via flask-migrate
 apscheduler==3.8.1
     # via -r requirements.in
 authlib==0.15.5
     # via -r requirements.in
+backports.zoneinfo==0.2.1
+    # via
+    #   pytz-deprecation-shim
+    #   tzlocal
 certifi==2020.12.5
     # via
     #   minio
@@ -24,13 +28,6 @@ cryptography==3.4.7
     # via
     #   authlib
     #   pymysql
-flask==2.0.2
-    # via
-    #   -r requirements.in
-    #   flask-login
-    #   flask-marshmallow
-    #   flask-migrate
-    #   flask-sqlalchemy
 flask-login==0.5.0
     # via -r requirements.in
 flask-marshmallow==0.14.0
@@ -41,6 +38,14 @@ flask-sqlalchemy==2.5.1
     # via
     #   -r requirements.in
     #   flask-migrate
+flask==2.0.2
+    # via
+    #   -r requirements.in
+    #   flask-login
+    #   flask-marshmallow
+    #   flask-migrate
+    #   flask-sqlalchemy
+    #   prometheus-flask-exporter
 greenlet==1.1.2
     # via sqlalchemy
 gunicorn==20.1.0
@@ -57,18 +62,22 @@ markupsafe==2.0.1
     # via
     #   jinja2
     #   mako
+marshmallow-sqlalchemy==0.26.1
+    # via -r requirements.in
 marshmallow==3.14.1
     # via
     #   flask-marshmallow
     #   marshmallow-sqlalchemy
-marshmallow-sqlalchemy==0.26.1
-    # via -r requirements.in
 minio==7.1.2
     # via -r requirements.in
 numpy==1.21.4
     # via pandas
 pandas==1.3.4
     # via -r requirements.in
+prometheus-client==0.12.0
+    # via prometheus-flask-exporter
+prometheus-flask-exporter==0.18.6
+    # via -r requirements.in
 pycparser==2.20
     # via cffi
 pymysql[rsa]==1.0.2
@@ -81,12 +90,12 @@ python-editor==1.0.4
     # via alembic
 python-http-client==3.3.3
     # via sendgrid
+pytz-deprecation-shim==0.1.0.post0
+    # via tzlocal
 pytz==2021.1
     # via
     #   apscheduler
     #   pandas
-pytz-deprecation-shim==0.1.0.post0
-    # via tzlocal
 requests==2.26.0
     # via -r requirements.in
 sendgrid==6.9.2