Use sane Celery defaults to prevent tasks from being delayed (#3267)

apache · Aug 10, 2017 · 08b7e89 · 08b7e89
1 parent 57421d1
commit 08b7e89
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/superset/cli.py b/superset/cli.py
@@ -194,7 +194,7 @@ def worker(workers):
         celery_app.conf.update(
             CELERYD_CONCURRENCY=config.get("SUPERSET_CELERY_WORKERS"))
 
-    worker = celery_worker.worker(app=celery_app)
+    worker = celery_app.Worker(optimization='fair')
     worker.run()
 
 

diff --git a/superset/config.py b/superset/config.py
@@ -245,6 +245,8 @@ class CeleryConfig(object):
   CELERY_RESULT_BACKEND = 'db+sqlite:///celery_results.sqlite'
   CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}}
   CELERYD_LOG_LEVEL = 'DEBUG'
+  CELERYD_PREFETCH_MULTIPLIER = 1
+  CELERY_ACKS_LATE = True
 CELERY_CONFIG = CeleryConfig
 """
 CELERY_CONFIG = None