From 3c98f6f2719b0b06b431ef1937ba90ae1a5b2ed9 Mon Sep 17 00:00:00 2001 From: Kevin Lu <6320810+kevinlul@users.noreply.github.com> Date: Fri, 3 Dec 2021 16:12:39 -0500 Subject: [PATCH 1/8] Add prometheus-flask-exporter --- flask/requirements-dev.txt | 22 +++++++++++++++------- flask/requirements.in | 1 + flask/requirements.txt | 17 +++++++++++------ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/flask/requirements-dev.txt b/flask/requirements-dev.txt index 316de3057..2a6e1ad50 100644 --- a/flask/requirements-dev.txt +++ b/flask/requirements-dev.txt @@ -32,12 +32,6 @@ cryptography==3.4.7 # pymysql debugpy==1.5.1 # via -r requirements-dev.in -flask==2.0.2 - # via - # -r requirements.in - # flask-login - # flask-migrate - # flask-sqlalchemy flask-login==0.5.0 # via -r requirements.in flask-migrate==3.1.0 @@ -46,6 +40,13 @@ flask-sqlalchemy==2.5.1 # via # -r requirements.in # flask-migrate +flask==2.0.2 + # via + # -r requirements.in + # flask-login + # flask-migrate + # flask-sqlalchemy + # prometheus-flask-exporter gunicorn==20.1.0 # via -r requirements.in idna==2.10 @@ -86,6 +87,10 @@ platformdirs==2.2.0 # pylint pluggy==0.13.1 # via pytest +prometheus-client==0.12.0 + # via prometheus-flask-exporter +prometheus-flask-exporter==0.18.6 + # via -r requirements.in py==1.10.0 # via pytest pycparser==2.20 @@ -124,7 +129,10 @@ toml==0.10.2 tomli==1.1.0 # via black typing-extensions==3.10.0.2 - # via black + # via + # astroid + # black + # pylint urllib3==1.26.5 # via # minio diff --git a/flask/requirements.in b/flask/requirements.in index eade74ac3..6c26094a4 100644 --- a/flask/requirements.in +++ b/flask/requirements.in @@ -6,6 +6,7 @@ flask_migrate gunicorn minio pandas +prometheus-flask-exporter pymysql[rsa] requests sqlalchemy diff --git a/flask/requirements.txt b/flask/requirements.txt index 6a2e26f8a..41a22195a 100644 --- a/flask/requirements.txt +++ b/flask/requirements.txt @@ -22,12 +22,6 @@ cryptography==3.4.7 # via # authlib # pymysql -flask==2.0.2 - # via - # -r requirements.in - # flask-login - # flask-migrate - # flask-sqlalchemy flask-login==0.5.0 # via -r requirements.in flask-migrate==3.1.0 @@ -36,6 +30,13 @@ flask-sqlalchemy==2.5.1 # via # -r requirements.in # flask-migrate +flask==2.0.2 + # via + # -r requirements.in + # flask-login + # flask-migrate + # flask-sqlalchemy + # prometheus-flask-exporter gunicorn==20.1.0 # via -r requirements.in idna==2.10 @@ -56,6 +57,10 @@ numpy==1.20.2 # via pandas pandas==1.3.4 # via -r requirements.in +prometheus-client==0.12.0 + # via prometheus-flask-exporter +prometheus-flask-exporter==0.18.6 + # via -r requirements.in pycparser==2.20 # via cffi pymysql[rsa]==1.0.2 From 76f0042429d13378b332f54ab4d72e019f7a19c3 Mon Sep 17 00:00:00 2001 From: Kevin Lu <6320810+kevinlul@users.noreply.github.com> Date: Fri, 3 Dec 2021 17:07:22 -0500 Subject: [PATCH 2/8] Draft Prometheus integration (needs PROMETHEUS_MULTIPROC_DIR) --- flask/app/__init__.py | 3 ++- flask/app/extensions.py | 2 ++ flask/gunicorn.conf.py | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 flask/gunicorn.conf.py diff --git a/flask/app/__init__.py b/flask/app/__init__.py index 9562fc830..cad11fa08 100644 --- a/flask/app/__init__.py +++ b/flask/app/__init__.py @@ -1,6 +1,6 @@ import logging from flask import Flask, logging as flask_logging -from .extensions import db, login, migrate, oauth +from .extensions import db, login, migrate, metrics, oauth from .utils import DateTimeEncoder from app import ( @@ -69,6 +69,7 @@ def register_extensions(app): server_metadata_url=app.config["OIDC_WELL_KNOWN"], client_kwargs={"scope": "openid"}, ) + metrics.init_app(app) def config_logger(app): diff --git a/flask/app/extensions.py b/flask/app/extensions.py index 2aecabebb..701b286ff 100644 --- a/flask/app/extensions.py +++ b/flask/app/extensions.py @@ -2,10 +2,12 @@ from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy from authlib.integrations.flask_client import OAuth +from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics db = SQLAlchemy() login = LoginManager() migrate = Migrate(compare_type=True) oauth = OAuth() +metrics = GunicornPrometheusMetrics(None, path=None) login.session_protection = "strong" diff --git a/flask/gunicorn.conf.py b/flask/gunicorn.conf.py new file mode 100644 index 000000000..d0f43fd89 --- /dev/null +++ b/flask/gunicorn.conf.py @@ -0,0 +1,9 @@ +from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics + + +def when_ready(server): + GunicornPrometheusMetrics.start_http_server_when_ready(8080) + + +def child_exit(server, worker): + GunicornPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid) From 5fcd9e3f16230d6ad83fe85e40924f65958fa11f Mon Sep 17 00:00:00 2001 From: Kevin Lu <6320810+kevinlul@users.noreply.github.com> Date: Thu, 9 Dec 2021 15:10:31 -0500 Subject: [PATCH 3/8] Fix preemptive requirements-dev dependency bump --- flask/requirements-dev.txt | 80 ++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 42 deletions(-) diff --git a/flask/requirements-dev.txt b/flask/requirements-dev.txt index 44edd9f5a..a36c79730 100644 --- a/flask/requirements-dev.txt +++ b/flask/requirements-dev.txt @@ -4,13 +4,13 @@ # # pip-compile requirements-dev.in # -alembic==1.7.5 +alembic==1.5.8 # via flask-migrate apscheduler==3.8.1 # via -r requirements.in astroid==2.9.0 # via pylint -attrs==21.2.0 +attrs==20.3.0 # via pytest authlib==0.15.5 # via -r requirements.in @@ -18,21 +18,21 @@ backports.zoneinfo==0.2.1 # via # pytz-deprecation-shim # tzlocal -black==21.12b0 +black==21.11b1 # via -r requirements-dev.in -certifi==2021.10.8 +certifi==2020.12.5 # via # minio # requests -cffi==1.15.0 +cffi==1.14.5 # via cryptography -charset-normalizer==2.0.9 +charset-normalizer==2.0.3 # via requests -click==8.0.3 +click==8.0.1 # via # black # flask -cryptography==36.0.0 +cryptography==3.4.7 # via # authlib # pymysql @@ -53,27 +53,21 @@ flask==2.0.2 # flask-migrate # flask-sqlalchemy # prometheus-flask-exporter -greenlet==1.1.2 - # via sqlalchemy gunicorn==20.1.0 # via -r requirements.in -idna==3.3 +idna==2.10 # via requests -importlib-metadata==4.8.2 - # via alembic -importlib-resources==5.4.0 - # via alembic iniconfig==1.1.1 # via pytest -isort==5.10.1 +isort==5.8.0 # via pylint itsdangerous==2.0.1 # via flask -jinja2==3.0.3 +jinja2==3.0.1 # via flask lazy-object-proxy==1.6.0 # via astroid -mako==1.1.6 +mako==1.1.4 # via alembic markupsafe==2.0.1 # via @@ -81,59 +75,65 @@ markupsafe==2.0.1 # mako mccabe==0.6.1 # via pylint -minio==7.1.2 +minio==7.1.1 # via -r requirements.in mypy-extensions==0.4.3 # via black -numpy==1.21.4 +numpy==1.21.3 # via pandas -packaging==21.3 +packaging==20.9 # via pytest pandas==1.3.4 # via -r requirements.in pathspec==0.9.0 # via black -platformdirs==2.4.0 +platformdirs==2.2.0 # via # black # pylint -pluggy==1.0.0 +pluggy==0.13.1 # via pytest prometheus-client==0.12.0 # via prometheus-flask-exporter prometheus-flask-exporter==0.18.6 # via -r requirements.in -py==1.11.0 +py==1.10.0 # via pytest -pycparser==2.21 +pycparser==2.20 # via cffi -pylint==2.12.2 +pylint==2.12.1 # via -r requirements-dev.in pymysql[rsa]==1.0.2 # via -r requirements.in -pyparsing==3.0.6 +pyparsing==2.4.7 # via packaging pytest==6.2.5 # via -r requirements-dev.in -python-dateutil==2.8.2 - # via pandas -python-http-client==3.3.4 +python-dateutil==2.8.1 + # via + # alembic + # pandas +python-editor==1.0.4 + # via alembic +python-http-client==3.3.3 # via sendgrid pytz-deprecation-shim==0.1.0.post0 # via tzlocal -pytz==2021.3 +pytz==2021.1 # via # apscheduler # pandas +regex==2021.4.4 + # via black requests==2.26.0 # via -r requirements.in sendgrid==6.9.2 # via -r requirements.in -six==1.16.0 +six==1.15.0 # via # apscheduler # python-dateutil -sqlalchemy==1.4.27 +sqlalchemy==1.3.24 # via # -r requirements.in # alembic @@ -144,9 +144,9 @@ toml==0.10.2 # via # pylint # pytest -tomli==1.2.2 +tomli==1.1.0 # via black -typing-extensions==4.0.1 +typing-extensions==3.10.0.2 # via # astroid # black @@ -155,18 +155,14 @@ tzdata==2021.5 # via pytz-deprecation-shim tzlocal==4.1 # via apscheduler -urllib3==1.26.7 +urllib3==1.26.5 # via # minio # requests -werkzeug==2.0.2 +werkzeug==2.0.1 # via flask -wrapt==1.13.3 +wrapt==1.12.1 # via astroid -zipp==3.6.0 - # via - # importlib-metadata - # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools From d62f8a56d2a01363dd84b80fad5826f9429328a0 Mon Sep 17 00:00:00 2001 From: Kevin Lu <6320810+kevinlul@users.noreply.github.com> Date: Thu, 9 Dec 2021 15:52:49 -0500 Subject: [PATCH 4/8] Integrate entrypoints into Dockerfile --- Dockerfile | 4 ++-- docker-compose.ccm.yaml | 2 -- docker-compose.cheo.yaml | 2 -- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 84f69a781..35ca45509 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,5 +17,5 @@ COPY . . ENV FLASK_ENV production EXPOSE 5000 # Prevent accidentally using this image for development by adding the prod server arguments in the entrypoint -ENTRYPOINT ["gunicorn", "wsgi:app", "--bind", "0.0.0.0:5000", "--access-logfile", "-", "--log-file", "-"] -CMD ["--preload", "--workers", "2", "--threads", "2"] +# Automatically run migrations on startup +ENTRYPOINT ["./utils/run.sh", "prod", "--bind", "0.0.0.0:5000", "--access-logfile", "-", "--log-file", "-"] diff --git a/docker-compose.ccm.yaml b/docker-compose.ccm.yaml index 7e8b845a2..c5c18110c 100644 --- a/docker-compose.ccm.yaml +++ b/docker-compose.ccm.yaml @@ -8,8 +8,6 @@ x-common: &common x-app: &app image: "ghcr.io/ccmbioinfo/stager:${ST_VERSION}" user: www-data - # Not in Dockerfile yet because the base image is used on stager-dev - entrypoint: ./utils/run.sh prod --bind 0.0.0.0:5000 --access-logfile - --log-file - command: --preload --workers ${GUNICORN_WORKERS:-1} healthcheck: test: ["CMD", "./healthcheck.py"] diff --git a/docker-compose.cheo.yaml b/docker-compose.cheo.yaml index 3f730f800..694d831a4 100644 --- a/docker-compose.cheo.yaml +++ b/docker-compose.cheo.yaml @@ -18,8 +18,6 @@ services: SENDGRID_FROM_EMAIL: ports: - "5000:5000" - # Not in Dockerfile yet because the base image is used on stager-dev - entrypoint: ./utils/run.sh prod --bind 0.0.0.0:5000 --access-logfile - --log-file - command: --preload --workers ${GUNICORN_WORKERS:-1} healthcheck: test: ["CMD", "./healthcheck.py"] From 94184922dde0b038bab429c3194b204b4f4a3afa Mon Sep 17 00:00:00 2001 From: Kevin Lu <6320810+kevinlul@users.noreply.github.com> Date: Thu, 9 Dec 2021 17:13:27 -0500 Subject: [PATCH 5/8] Initialize metrics handler appropriately per environment --- flask/app/__init__.py | 1 + flask/app/extensions.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/flask/app/__init__.py b/flask/app/__init__.py index 2e89ffcfa..4533f61b9 100644 --- a/flask/app/__init__.py +++ b/flask/app/__init__.py @@ -90,6 +90,7 @@ def register_extensions(app): client_kwargs={"scope": "openid"}, ) metrics.init_app(app) + metrics.info("stager", "Stager process info", revision=app.config.get("GIT_SHA")) def config_logger(app): diff --git a/flask/app/extensions.py b/flask/app/extensions.py index 701b286ff..047f641ad 100644 --- a/flask/app/extensions.py +++ b/flask/app/extensions.py @@ -1,13 +1,21 @@ +from os import getenv from flask_login import LoginManager from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy from authlib.integrations.flask_client import OAuth +from prometheus_flask_exporter import PrometheusMetrics from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics db = SQLAlchemy() login = LoginManager() migrate = Migrate(compare_type=True) oauth = OAuth() -metrics = GunicornPrometheusMetrics(None, path=None) + +if getenv("FLASK_ENV") == "development": + # Note that /metrics will not be live without DEBUG_METRICS set + metrics = PrometheusMetrics(None) +else: # production + # Must additionally set PROMETHEUS_MULTIPROC_DIR + metrics = GunicornPrometheusMetrics(None, path=None) login.session_protection = "strong" From ab4ae39c530b27b081e814e97e9623ca332f1b53 Mon Sep 17 00:00:00 2001 From: Kevin Lu <6320810+kevinlul@users.noreply.github.com> Date: Thu, 9 Dec 2021 17:14:04 -0500 Subject: [PATCH 6/8] Disable pip cache when building images, set Prometheus multiprocess directory --- Dockerfile | 5 +++-- flask/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 35ca45509..613e8fe73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,11 +11,12 @@ ENV GIT_SHA=${GIT_SHA} WORKDIR /usr/src/stager # Install PyPI prod-only packages first and then copy the MinIO client as the latter updates more frequently COPY requirements.txt . -RUN pip3 install -r requirements.txt +RUN pip3 install --no-cache-dir -r requirements.txt COPY --from=mc /usr/bin/mc /usr/bin/mc COPY . . ENV FLASK_ENV production -EXPOSE 5000 +ENV PROMETHEUS_MULTIPROC_DIR /tmp +EXPOSE 5000 8080 # Prevent accidentally using this image for development by adding the prod server arguments in the entrypoint # Automatically run migrations on startup ENTRYPOINT ["./utils/run.sh", "prod", "--bind", "0.0.0.0:5000", "--access-logfile", "-", "--log-file", "-"] diff --git a/flask/Dockerfile b/flask/Dockerfile index 5d63e0655..ab118ab2c 100644 --- a/flask/Dockerfile +++ b/flask/Dockerfile @@ -8,7 +8,7 @@ LABEL org.opencontainers.image.vendor Centre for Computational Medicine WORKDIR /usr/src/stager # Install PyPI packages first and then copy the MinIO client as the latter updates more frequently COPY requirements-dev.txt . -RUN pip3 install -r requirements-dev.txt +RUN pip3 install --no-cache-dir -r requirements-dev.txt COPY --from=mc /usr/bin/mc /usr/bin/mc ENV FLASK_ENV development # Prevent accidentally running this image in production. From f3d191c3a7116f248554de59349d816804c8a16a Mon Sep 17 00:00:00 2001 From: Kevin Lu <6320810+kevinlul@users.noreply.github.com> Date: Thu, 9 Dec 2021 17:14:34 -0500 Subject: [PATCH 7/8] Add alternate gunicorn configuration for testing metrics setup --- docker-compose.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 4262c2bd5..051003147 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -58,6 +58,7 @@ services: MINIO_REGION_NAME: FLASK_DEBUG: 1 PYTHONDONTWRITEBYTECODE: 1 + DEBUG_METRICS: 1 ports: - "${FLASK_HOST_PORT:-127.0.0.1:5000}:5000" - "${DEBUG_HOST_PORT:-127.0.0.1:5678}:5678" @@ -67,3 +68,26 @@ services: volumes: - ./flask:/usr/src/stager entrypoint: ["python", "-m", "debugpy", "--listen", "0.0.0.0:5678", "-m", "flask", "run", "--host=0.0.0.0"] + # Simulated production gunicorn configuration + app_gunicorn: + build: + context: flask + dockerfile: ../Dockerfile + args: + GIT_SHA: + image: ghcr.io/ccmbioinfo/stager:latest + profiles: + - gunicorn + environment: + ST_SECRET_KEY: + ST_DATABASE_URI: "mysql+pymysql://${MYSQL_USER}:${MYSQL_PASSWORD}@mysql/${MYSQL_DATABASE}" + MINIO_ENDPOINT: minio:9000 + MINIO_ACCESS_KEY: + MINIO_SECRET_KEY: + MINIO_REGION_NAME: + ports: + - "${FLASK_HOST_PORT:-127.0.0.1:5000}:5000" + - "${METRICS_HOST_PORT:-127.0.0.1:8080}:8080" + depends_on: + - mysql + - minio From 95520683a70e39acc26d26cee352fbeaa02e8ea0 Mon Sep 17 00:00:00 2001 From: Kevin Lu <6320810+kevinlul@users.noreply.github.com> Date: Fri, 10 Dec 2021 10:52:37 -0500 Subject: [PATCH 8/8] Open up designated metrics port in CHEO-RI --- docker-compose.cheo.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.cheo.yaml b/docker-compose.cheo.yaml index 694d831a4..67447164c 100644 --- a/docker-compose.cheo.yaml +++ b/docker-compose.cheo.yaml @@ -18,6 +18,7 @@ services: SENDGRID_FROM_EMAIL: ports: - "5000:5000" + - "9121:8080" command: --preload --workers ${GUNICORN_WORKERS:-1} healthcheck: test: ["CMD", "./healthcheck.py"]