From c83e40b0474d925ae746fe556d4f3ac0bbbfacd6 Mon Sep 17 00:00:00 2001
From: Omer Lachish <omer@rauchy.net>
Date: Wed, 17 Jul 2019 10:38:56 +0300
Subject: [PATCH] Celery doesn't auto reload in development (#3898)

* pick up *.py file changes and restart scheduler

* only watch /redash in order to avoid reloading on other file changes (e.g. tests)

* add dev_scheduler entrypoint

* use exec

* Update bin/docker-entrypoint

* rename dev_scheduler to dev_worker

* use same defaults as worker
---
 bin/docker-entrypoint | 15 +++++++++++++++
 docker-compose.yml    |  8 +++++---
 requirements_dev.txt  |  1 +
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/bin/docker-entrypoint b/bin/docker-entrypoint
index 62d80e9526..2438f4621c 100755
--- a/bin/docker-entrypoint
+++ b/bin/docker-entrypoint
@@ -20,6 +20,16 @@ scheduler() {
   exec /usr/local/bin/celery worker --app=redash.worker --beat -s$SCHEDULE_DB -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair
 }
 
+dev_worker() {
+  WORKERS_COUNT=${WORKERS_COUNT:-2}
+  QUEUES=${QUEUES:-queries,scheduled_queries,celery,schemas}
+  SCHEDULE_DB=${SCHEDULE_DB:-celerybeat-schedule}
+
+  echo "Starting dev scheduler and $WORKERS_COUNT workers for queues: $QUEUES..."
+
+  exec watchmedo auto-restart --directory=./redash/ --pattern=*.py --recursive -- /usr/local/bin/celery worker --app=redash.worker --beat -s$SCHEDULE_DB -c$WORKERS_COUNT -Q$QUEUES -linfo --max-tasks-per-child=10 -Ofair
+}
+
 server() {
   exec /usr/local/bin/gunicorn -b 0.0.0.0:5000 --name redash -w${REDASH_WEB_WORKERS:-4} redash.wsgi:app
 }
@@ -41,6 +51,7 @@ help() {
   echo "server -- start Redash server (with gunicorn)"
   echo "worker -- start Celery worker"
   echo "scheduler -- start Celery worker with a beat (scheduler) process"
+  echo "dev_worker  -- start Celery worker with a beat (scheduler) process which picks up code changes and reloads"
   echo "celery_healthcheck -- runs a Celery healthcheck. Useful for Docker's HEALTHCHECK mechanism."
   echo ""
   echo "shell -- open shell"
@@ -75,6 +86,10 @@ case "$1" in
     shift
     scheduler
     ;;
+  dev_worker)
+    shift
+    dev_worker
+    ;;
   dev_server)
     export FLASK_DEBUG=1
     exec /app/manage.py runserver --debugger --reload -h 0.0.0.0
diff --git a/docker-compose.yml b/docker-compose.yml
index 8bd2c4dc85..c8ed6c887f 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,6 +1,6 @@
 # This configuration file is for the **development** setup.
 # For a production example please refer to setup/docker-compose.yml.
-version: '3'
+version: '3.7'
 services:
   server:
     build: .
@@ -21,9 +21,11 @@ services:
       REDASH_RATELIMIT_ENABLED: "false"
   worker:
     build: .
-    command: scheduler
+    command: dev_worker
     volumes:
-      - ".:/app"
+      - type: bind
+        source: .
+        target: /app
     depends_on:
       - server
     environment:
diff --git a/requirements_dev.txt b/requirements_dev.txt
index dc33df9a77..7e6b60a9e5 100644
--- a/requirements_dev.txt
+++ b/requirements_dev.txt
@@ -10,3 +10,4 @@ botocore==1.12.115
 PyAthena>=1.5.0
 ptvsd==4.2.3
 freezegun==0.3.11
+watchdog==0.9.0