From d3a90ad5ed474c8a14b6de5dd7260cc251b66dc8 Mon Sep 17 00:00:00 2001 From: pharr117 Date: Wed, 11 Oct 2023 18:49:02 -0400 Subject: [PATCH 1/4] Rework main application to add subcommands that allow for 3 different app modes: 1. debug mode runs the app in a single build, the data is cached in process memory and the Flask app is served at the same time 2. reader mode runs just the Flask app and it reads the cache from a Filesystem cache 3. writer mode runs just the update thread cycle which loops continuosly and writes to a Filesystem cache Running reader and writer fpointing to the same filesystem cache dir should be safe for simple production use --- app.py | 41 ++++++++++++++++++++++++++++++++++++++--- requirements.txt | 1 + 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index 32a0726..6fd1689 100644 --- a/app.py +++ b/app.py @@ -15,6 +15,7 @@ import json import subprocess import semantic_version +import argparse app = Flask(__name__) @@ -750,6 +751,7 @@ def start_update_data_thread(): update_thread = threading.Thread(target=update_data) update_thread.daemon = True update_thread.start() + return update_thread @app.route("/healthz") @@ -786,7 +788,40 @@ def get_testnet_data(): ) +def get_app_args(): + # setup the application args, providing 3 different commands that run the application in different modes + parser = argparse.ArgumentParser(description="Cosmos upgrade monitor") + + app_role_subparsers = parser.add_subparsers(title='subcommands', + description='valid app roles to run', + help='runs the application in various configurations', dest='command') + + # debug mode runs the application as a development server, with the cache setup as a simple in Python cache + app_role_subparsers.add_parser('debug') + # reader mode runs the application as a production Flask server, with the cache setup as an external store that is read from + reader_parser = app_role_subparsers.add_parser('reader') + # writer mode runs the application as a datastore writer, with the cache setup as an external store that is written to + writer_parser = app_role_subparsers.add_parser('writer') + + # add the filesystem cache location arg to reader and writer parsers + reader_parser.add_argument('--cache-location', dest='cache_location', default='/tmp/cosmos_upgrade_monitor_cache') + writer_parser.add_argument('--cache-location', dest='cache_location', default='/tmp/cosmos_upgrade_monitor_cache') + + return parser.parse_args() + if __name__ == "__main__": - app.debug = True - start_update_data_thread() - app.run(host="0.0.0.0", use_reloader=False) + args = get_app_args() + if args.command == "debug": + print("Running in debug mode") + app.debug = True + start_update_data_thread() + app.run(host="0.0.0.0", use_reloader=False) + elif args.command == "reader": + cache = Cache(app, config={"CACHE_TYPE": "FileSystemCache", "CACHE_DIR": args.cache_location}) + print("Running as reader") + app.run(host="0.0.0.0", use_reloader=False) + elif args.command == "writer": + cache = Cache(app, config={"CACHE_TYPE": "FileSystemCache", "CACHE_DIR": args.cache_location}) + print("Running as writer") + thread = start_update_data_thread() + thread.join() diff --git a/requirements.txt b/requirements.txt index 3853af5..9f49bed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ requests==2.26.0 urllib3==1.26.7 gunicorn==20.1.0 semantic-version==2.10.0 +Werkzeug==2.2.2 From 7c09bc3b241948310e5d4cce763efcf1f10ada31 Mon Sep 17 00:00:00 2001 From: pharr117 Date: Wed, 11 Oct 2023 19:11:15 -0400 Subject: [PATCH 2/4] Move flask app returning into factories for debug and production modes (can be used by gunicorn now) --- app.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/app.py b/app.py index 6fd1689..0e4e141 100644 --- a/app.py +++ b/app.py @@ -71,7 +71,7 @@ def get_chain_watch_env_var(): return chain_watch -CHAIN_WATCH = get_chain_watch_env_var() +CHAIN_WATCH = [] # Clone the repo @@ -809,19 +809,36 @@ def get_app_args(): return parser.parse_args() +# Acts as a factory, returning the flask app with the cache configured as FSCache +# Can be called by gunicorn like so: +# gunicorn -b 0.0.0.0 -w 4 'app:get_production_flask_app("/tmp/cosmos_upgrade_monitor_cache")' +def get_production_flask_app(cache_location): + global cache + cache = Cache(app, config={"CACHE_TYPE": "FileSystemCache", "CACHE_DIR": cache_location}) + return app + +# Acts as a factory, returning the flask app with the cache configured as Simple +def get_debug_flask_app(): + global cache + cache = Cache(app, config={"CACHE_TYPE": "SimpleCache"}) + app.debug = True + return app + if __name__ == "__main__": args = get_app_args() if args.command == "debug": + CHAIN_WATCH = get_chain_watch_env_var() print("Running in debug mode") - app.debug = True start_update_data_thread() + debug_app = get_debug_flask_app() app.run(host="0.0.0.0", use_reloader=False) elif args.command == "reader": - cache = Cache(app, config={"CACHE_TYPE": "FileSystemCache", "CACHE_DIR": args.cache_location}) - print("Running as reader") - app.run(host="0.0.0.0", use_reloader=False) + print("Running Flask application with FSCache") + app = get_production_flask_app(args.cache_location) + app.run() elif args.command == "writer": cache = Cache(app, config={"CACHE_TYPE": "FileSystemCache", "CACHE_DIR": args.cache_location}) + CHAIN_WATCH = get_chain_watch_env_var() print("Running as writer") thread = start_update_data_thread() thread.join() From 4cdd7f74fa0ac20d38c3ead2afa2d8d4f0927d5c Mon Sep 17 00:00:00 2001 From: pharr117 Date: Wed, 11 Oct 2023 19:28:04 -0400 Subject: [PATCH 3/4] Update Dockerfile and add docker-compose.yaml for a production ready build --- Dockerfile | 4 ---- docker-compose.yaml | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 docker-compose.yaml diff --git a/Dockerfile b/Dockerfile index 61c7427..1ba668e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,3 @@ COPY . /app/ # Expose port 5000 for the Flask app to listen on when running within the container EXPOSE 5000 - -# Define the command to start the container. Use gunicorn as the WSGI server to serve the Flask app -# CMD ["gunicorn", "--bind", "0.0.0.0:5000", "app:app"] -CMD ["python", "app.py"] diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..9f89bbf --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,18 @@ +services: + reader: + build: + context: . + ports: + - "5000:5000" + volumes: + - cache-volume:/var/lib/cache + command: ["gunicorn", "--bind", "0.0.0.0:5000", "-w", "4", "app:get_production_flask_app(\"/var/lib/cache\")"] + writer: + build: + context: . + volumes: + - cache-volume:/var/lib/cache + command: ["python", "app.py", "writer", "--cache-location", "/var/lib/cache"] + +volumes: + cache-volume: \ No newline at end of file From fc91609c73d97e437fd1c1b8f1d0129525c294a5 Mon Sep 17 00:00:00 2001 From: pharr117 Date: Wed, 11 Oct 2023 19:30:21 -0400 Subject: [PATCH 4/4] Precommit --- app.py | 2 +- docker-compose.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 0e4e141..df0da57 100644 --- a/app.py +++ b/app.py @@ -795,7 +795,7 @@ def get_app_args(): app_role_subparsers = parser.add_subparsers(title='subcommands', description='valid app roles to run', help='runs the application in various configurations', dest='command') - + # debug mode runs the application as a development server, with the cache setup as a simple in Python cache app_role_subparsers.add_parser('debug') # reader mode runs the application as a production Flask server, with the cache setup as an external store that is read from diff --git a/docker-compose.yaml b/docker-compose.yaml index 9f89bbf..ba2c655 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,6 @@ services: reader: - build: + build: context: . ports: - "5000:5000" @@ -8,11 +8,11 @@ services: - cache-volume:/var/lib/cache command: ["gunicorn", "--bind", "0.0.0.0:5000", "-w", "4", "app:get_production_flask_app(\"/var/lib/cache\")"] writer: - build: + build: context: . volumes: - cache-volume:/var/lib/cache command: ["python", "app.py", "writer", "--cache-location", "/var/lib/cache"] volumes: - cache-volume: \ No newline at end of file + cache-volume: