add syncstudytables management command (#1509)

CHANGELOG.rst

@@ -18,6 +18,7 @@ Added
 - **Samplesheets**
     - Mac keyboard shortcut support for multi-cell copying (#1531)
     - Study render table caching (#1509)
+    - ``syncstudytables`` management command (#1509)
 
 Changed
 -------

docs_manual/source/admin_commands.rst

@@ -56,3 +56,8 @@ operations regarding sample sheets, landing zones, iRODS data and ontologies.
     Find orphans in iRODS project collections.
 ``syncnames``
     Synchronize alternative names for sample sheet material search.
+``syncstudytables``
+    Build study render tables in cache for all study tables. These will be
+    automatically built when accessing sample sheets if existing  cache is not
+    up-to-date, but this can be used to e.g. regenerate the cache if something
+    has been changed in study table rendering.

samplesheets/management/commands/syncstudytables.py

@@ -0,0 +1,131 @@
+"""Syncstudytables management command"""
+
+import sys
+
+from django.core.management.base import BaseCommand
+
+# Projectroles dependency
+from projectroles.management.logging import ManagementCommandLogger
+from projectroles.models import Project, SODAR_CONSTANTS
+from projectroles.plugins import get_backend_api
+
+from samplesheets.models import Investigation
+from samplesheets.rendering import (
+    SampleSheetTableBuilder,
+    STUDY_TABLE_CACHE_ITEM,
+)
+
+
+logger = ManagementCommandLogger(__name__)
+table_builder = SampleSheetTableBuilder()
+
+
+# SODAR constants
+PROJECT_TYPE_PROJECT = SODAR_CONSTANTS['PROJECT_TYPE_PROJECT']
+# Local constants
+APP_NAME = 'samplesheets'
+
+
+class Command(BaseCommand):
+    help = 'Syncs study render tables in sodarcache for optimized rendering'
+
+    @classmethod
+    def _get_log_project(cls, project):
+        """Return logging-friendly project title"""
+        return '"{}" ({})'.format(project.title, project.sodar_uuid)
+
+    @classmethod
+    def _get_log_study(cls, study):
+        """Return logging-friendly project title"""
+        return '"{}" ({})'.format(study.get_title(), study.sodar_uuid)
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-p',
+            '--project',
+            metavar='UUID',
+            type=str,
+            help='Limit sync to a project',
+        )
+
+    def handle(self, *args, **options):
+        cache_backend = get_backend_api('sodar_cache')
+        if not cache_backend:
+            logger.error('Sodarcache not enabled, exiting')
+            sys.exit(1)
+
+        q_kwargs = {'type': PROJECT_TYPE_PROJECT}
+        if options.get('project'):
+            q_kwargs['sodar_uuid'] = options['project']
+        projects = Project.objects.filter(**q_kwargs).order_by('full_title')
+        if not projects:
+            logger.info(
+                'No project{} found'.format(
+                    's' if not options.get('project') else ''
+                )
+            )
+            return
+        if options.get('project'):
+            project = projects.first()
+            logger.info(
+                'Limiting sync to project {}'.format(
+                    self._get_log_project(project)
+                )
+            )
+
+        for project in projects:
+            study_count = 0
+            try:
+                investigation = Investigation.objects.get(
+                    project=project, active=True
+                )
+            except Investigation.DoesNotExist:
+                logger.debug(
+                    'No investigation found, skipping for project {}'.format(
+                        self._get_log_project(project)
+                    )
+                )
+                continue
+            logger.debug(
+                'Building study render tables for project {}..'.format(
+                    self._get_log_project(project)
+                )
+            )
+            for study in investigation.studies.all():
+                try:
+                    study_tables = table_builder.build_study_tables(
+                        study, use_config=True
+                    )
+                except Exception as ex:
+                    logger.error(
+                        'Error building tables for study {}: {}'.format(
+                            self._get_log_study(study), ex
+                        )
+                    )
+                    continue
+                item_name = STUDY_TABLE_CACHE_ITEM.format(
+                    study=study.sodar_uuid
+                )
+                try:
+                    cache_backend.set_cache_item(
+                        app_name=APP_NAME,
+                        name=item_name,
+                        data=study_tables,
+                        project=project,
+                    )
+                    logger.info('Set cache item "{}"'.format(item_name))
+                    study_count += 1
+                except Exception as ex:
+                    logger.error(
+                        'Failed to set cache item "{}": {}'.format(
+                            item_name, ex
+                        )
+                    )
+            logger.info(
+                'Built {} study table{} for project {}'.format(
+                    study_count,
+                    's' if study_count != 1 else '',
+                    self._get_log_project(project),
+                )
+            )
+        logger.info('Study table cache sync done')

samplesheets/plugins.py

@@ -625,6 +625,9 @@ def update_cache(self, name=None, project=None, user=None):
         :raise: Exception if required backends (sodar_cache and omics_irods)
                 are not found.
         """
+        # NOTE: This will not sync cached study render tables, they are created
+        #       synchronously upon access if not up-to-date. To sync the cache
+        #       for all study tables, use the syncstudytables command.
         cache_backend = get_backend_api('sodar_cache')
         irods_backend = get_backend_api('omics_irods')
         if not cache_backend or not irods_backend:

samplesheets/tests/test_commands.py

@@ -1,14 +1,21 @@
 """Tests for management commands in the samplesheets app"""
 
+import uuid
+
 from django.core.management import call_command
 
 from test_plus.test import TestCase
 
 # Projectroles dependency
 from projectroles.models import Role, SODAR_CONSTANTS
+from projectroles.plugins import get_backend_api
 from projectroles.tests.test_models import ProjectMixin, RoleAssignmentMixin
 
+# Sodarcache dependency
+from sodarcache.models import JSONCacheItem
+
 from samplesheets.models import GenericMaterial
+from samplesheets.rendering import STUDY_TABLE_CACHE_ITEM
 from samplesheets.tests.test_io import (
     SampleSheetIOMixin,
     SHEET_DIR,
@@ -18,6 +25,7 @@
 
 
 # Local constants
+APP_NAME = 'samplesheets'
 SHEET_PATH = SHEET_DIR + 'i_small.zip'
 SHEET_PATH_INSERTED = SHEET_DIR_SPECIAL + 'i_small_insert.zip'
 SHEET_PATH_ALT = SHEET_DIR + 'i_small2.zip'
@@ -32,31 +40,123 @@ class TestSyncnamesCommand(
     def setUp(self):
         # Make owner user
         self.user_owner = self.make_user('owner')
-
         # Init project, role and assignment
         self.project = self._make_project(
             'TestProject', SODAR_CONSTANTS['PROJECT_TYPE_PROJECT'], None
         )
         self.role_owner = Role.objects.get_or_create(
             name=SODAR_CONSTANTS['PROJECT_ROLE_OWNER']
         )[0]
-        self.assignment_owner = self._make_assignment(
+        self.owner_as = self._make_assignment(
             self.project, self.user_owner, self.role_owner
         )
-
         # Import investigation
         self.investigation = self.import_isa_from_file(SHEET_PATH, self.project)
         self.study = self.investigation.studies.first()
-
         # Clear alt names from imported materials
         for m in GenericMaterial.objects.all():
             m.alt_names = ALT_NAMES_INVALID
             m.save()
 
     def test_command(self):
-        """Test syncnames command"""
+        """Test syncnames"""
         for m in GenericMaterial.objects.all():
             self.assertEqual(m.alt_names, ALT_NAMES_INVALID)
         call_command('syncnames')
         for m in GenericMaterial.objects.all():
             self.assertEqual(m.alt_names, get_alt_names(m.name))
+
+
+class TestSyncstudytablesCommand(
+    ProjectMixin, RoleAssignmentMixin, SampleSheetIOMixin, TestCase
+):
+    """Tests for the syncstudytables command"""
+
+    def setUp(self):
+        # Init user and role
+        self.user_owner = self.make_user('owner')
+        self.role_owner = Role.objects.get_or_create(
+            name=SODAR_CONSTANTS['PROJECT_ROLE_OWNER']
+        )[0]
+
+        # Init project
+        self.project = self._make_project(
+            'TestProject', SODAR_CONSTANTS['PROJECT_TYPE_PROJECT'], None
+        )
+        self.owner_as = self._make_assignment(
+            self.project, self.user_owner, self.role_owner
+        )
+        self.investigation = self.import_isa_from_file(SHEET_PATH, self.project)
+        self.study = self.investigation.studies.first()
+
+        # Init second project
+        self.project2 = self._make_project(
+            'TestProject2', SODAR_CONSTANTS['PROJECT_TYPE_PROJECT'], None
+        )
+        self.owner_as2 = self._make_assignment(
+            self.project2, self.user_owner, self.role_owner
+        )
+        self.investigation2 = self.import_isa_from_file(
+            SHEET_PATH_ALT, self.project2
+        )
+        self.study2 = self.investigation2.studies.first()
+
+        # Init helpers
+        self.cache_name = STUDY_TABLE_CACHE_ITEM.format(
+            study=self.study.sodar_uuid
+        )
+        self.cache_name2 = STUDY_TABLE_CACHE_ITEM.format(
+            study=self.study2.sodar_uuid
+        )
+        self.cache_args = [APP_NAME, self.cache_name, self.project]
+        self.cache_args2 = [APP_NAME, self.cache_name2, self.project2]
+        self.cache_backend = get_backend_api('sodar_cache')
+
+    def test_sync_all(self):
+        """Test syncstudytables for all projects"""
+        self.assertEqual(JSONCacheItem.objects.count(), 0)
+        call_command('syncstudytables')
+        self.assertEqual(JSONCacheItem.objects.count(), 2)
+        cache_item = self.cache_backend.get_cache_item(*self.cache_args)
+        self.assertIsInstance(cache_item, JSONCacheItem)
+        self.assertNotEqual(cache_item.data, {})
+        cache_item2 = self.cache_backend.get_cache_item(*self.cache_args2)
+        self.assertIsInstance(cache_item2, JSONCacheItem)
+        self.assertNotEqual(cache_item2.data, {})
+
+    def test_sync_all_existing(self):
+        """Test syncstudytables for all projects with existing items"""
+        self.cache_backend.set_cache_item(
+            APP_NAME, self.cache_name, {}, project=self.project
+        )
+        self.cache_backend.set_cache_item(
+            APP_NAME, self.cache_name2, {}, project=self.project2
+        )
+
+        self.assertEqual(JSONCacheItem.objects.count(), 2)
+        call_command('syncstudytables')
+        self.assertEqual(JSONCacheItem.objects.count(), 2)
+        cache_item = self.cache_backend.get_cache_item(*self.cache_args)
+        self.assertIsInstance(cache_item, JSONCacheItem)
+        self.assertNotEqual(cache_item.data, {})
+        cache_item2 = self.cache_backend.get_cache_item(*self.cache_args2)
+        self.assertIsInstance(cache_item2, JSONCacheItem)
+        self.assertNotEqual(cache_item2.data, {})
+
+    def test_sync_limit(self):
+        """Test syncstudytables limiting for single project"""
+        self.assertEqual(JSONCacheItem.objects.count(), 0)
+        call_command('syncstudytables', project=str(self.project.sodar_uuid))
+        self.assertEqual(JSONCacheItem.objects.count(), 1)
+        cache_item = self.cache_backend.get_cache_item(*self.cache_args)
+        self.assertIsInstance(cache_item, JSONCacheItem)
+        self.assertNotEqual(cache_item.data, {})
+        cache_item2 = self.cache_backend.get_cache_item(*self.cache_args2)
+        self.assertIsNone(cache_item2)
+
+    def test_sync_limit_invalid_project(self):
+        """Test syncstudytables limiting for non-existent project"""
+        self.assertEqual(JSONCacheItem.objects.count(), 0)
+        invalid_uuid = uuid.uuid4()
+        call_command('syncstudytables', project=str(invalid_uuid))
+        self.assertEqual(JSONCacheItem.objects.count(), 0)