From b5918dfc2d54dfa8c0091b8c3cf233e5862edd1f Mon Sep 17 00:00:00 2001 From: Daniel Chiquito Date: Mon, 27 Sep 2021 10:34:49 -0400 Subject: [PATCH 1/2] Add environment setting for Celery concurrency Some tasks are too memory intensive for celery, so having a setting is useful for tuning staging/production without releasing a new version. --- dandiapi/settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dandiapi/settings.py b/dandiapi/settings.py index 8fb789213..5613f4c2b 100644 --- a/dandiapi/settings.py +++ b/dandiapi/settings.py @@ -75,6 +75,10 @@ def before_binding(configuration: Type[ComposedConfiguration]): 'DEFAULT_AUTO_SCHEMA_CLASS': 'dandiapi.swagger.DANDISwaggerAutoSchema', } + # Some tasks working with lots of data need lots of memory, so we need to artificially lower + # the number of concurrent tasks (default is 8) to keep memory usage down. + CELERY_WORKER_CONCURRENCY = values.IntegerValue(environ=True, default=8) + class DevelopmentConfiguration(DandiMixin, DevelopmentBaseConfiguration): # This makes pydantic model schema allow URLs with localhost in them. @@ -106,7 +110,3 @@ class HerokuProductionConfiguration(DandiMixin, HerokuProductionBaseConfiguratio # the API server is running in (production/local or staging). class HerokuStagingConfiguration(HerokuProductionConfiguration): OAUTH2_PROVIDER_APPLICATION_MODEL = 'api.StagingApplication' - - # We are using cheaper Heroku dynos for staging, so we need to artificially lower the number - # of concurrent tasks (default is 8) to keep memory usage down. - CELERY_WORKER_CONCURRENCY = 2 From b07e02009e6bff1ebb456d1a7fba2aa2202411b0 Mon Sep 17 00:00:00 2001 From: Daniel Chiquito Date: Mon, 27 Sep 2021 10:58:17 -0400 Subject: [PATCH 2/2] Pin dandischema dependency This fixes an error where the latest dandischema release changed it's API and broke a feature. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index efb5e2f29..6f62e49d8 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ include_package_data=True, install_requires=[ 'celery', - 'dandischema>=0.3.4', + 'dandischema==0.3.4', 'django>=3.1.2', 'django-admin-display', 'django-allauth',