Bypass data type checks

bims/api_views/checklist.py

@@ -12,6 +12,8 @@
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
+from django.contrib.auth import get_user_model
+
 from bims.api_views.search import CollectionSearch
 from bims.enums import TaxonomicRank
 from bims.models.taxonomy import Taxonomy
@@ -33,6 +35,8 @@
     'park_or_mpa_name': 'Park or MPA name'
 }
 
+User = get_user_model()
+
 
 def get_custom_header(fieldnames, header_title_dict):
     custom_header = []
@@ -48,6 +52,17 @@ def get_serializer_keys(serializer_class):
     return list(serializer_class().get_fields().keys())
 
 
+def checklist_collection_records(download_request: DownloadRequest):
+    filters = parse_url_to_filters(
+        download_request.dashboard_url
+    )
+    search = CollectionSearch(
+        filters,
+        bypass_data_type_checks=True)
+    collection_records = search.process_search()
+    return collection_records
+
+
 def generate_checklist(download_request_id):
     if not download_request_id:
         return False
@@ -66,11 +81,10 @@ def generate_checklist(download_request_id):
     if not download_request:
         return False
 
-    filters = parse_url_to_filters(download_request.dashboard_url)
-
-    search = CollectionSearch(filters)
     batch_size = 1000
-    collection_records = search.process_search()
+    collection_records = checklist_collection_records(
+        download_request
+    )
     module_name = ''
 
     if collection_records.count() > 0:

bims/api_views/search.py

@@ -123,11 +123,13 @@ class CollectionSearch(object):
     site = None
     taxon_groups = TaxonGroup.objects.all()
     start_time = None
+    bypass_data_type_checks = False
 
-    def __init__(self, parameters, requester_id=None):
+    def __init__(self, parameters, requester_id=None, bypass_data_type_checks=False):
         self.parameters = dict(parameters)
         if requester_id:
             self.parameters['requester'] = requester_id
+        self.bypass_data_type_checks = bypass_data_type_checks
         super(CollectionSearch, self).__init__()
 
     def get_request_data(self, field, default_value=None):
@@ -632,18 +634,6 @@ def process_search(self):
 
         requester_id = self.parameters.get('requester', None)
 
-        is_sensitive_data_access_allowed = False
-        is_private_data_access_allowed = False
-        try:
-            requester = get_user_model().objects.get(id=requester_id)
-            user_groups = requester.groups.values_list('name', flat=True)
-            if 'SensitiveDataGroup' in user_groups:
-                is_sensitive_data_access_allowed = True
-            if 'PrivateDataGroup' in user_groups:
-                is_private_data_access_allowed = True
-        except get_user_model().DoesNotExist:
-            pass
-
         bio = bio.filter(
             Q(owner_id=requester_id) |
             Q(
@@ -652,16 +642,28 @@ def process_search(self):
             )
         )
 
-        accessible_data_types = ['public']
-        if is_sensitive_data_access_allowed:
-            accessible_data_types.append('sensitive')
-        if is_private_data_access_allowed:
-            accessible_data_types.append('private')
+        if not self.bypass_data_type_checks:
+            is_sensitive_data_access_allowed = False
+            is_private_data_access_allowed = False
+            try:
+                requester = get_user_model().objects.get(id=requester_id)
+                user_groups = requester.groups.values_list('name', flat=True)
+                if 'SensitiveDataGroup' in user_groups:
+                    is_sensitive_data_access_allowed = True
+                if 'PrivateDataGroup' in user_groups:
+                    is_private_data_access_allowed = True
+            except get_user_model().DoesNotExist:
+                pass
+            accessible_data_types = ['public']
+            if is_sensitive_data_access_allowed:
+                accessible_data_types.append('sensitive')
+            if is_private_data_access_allowed:
+                accessible_data_types.append('private')
 
-        bio = bio.filter(
-            Q(data_type='') |
-            Q(data_type__in=accessible_data_types)
-        )
+            bio = bio.filter(
+                Q(data_type='') |
+                Q(data_type__in=accessible_data_types)
+            )
 
         # Filter collection record with SASS Accreditation status
         validated_values = self.parse_request_json('validated')

bims/serializers/checklist_serializer.py

@@ -157,7 +157,6 @@ class Meta:
         ]
 
 
-
 class ChecklistSerializer(ChecklistBaseSerializer):
     """
     Serializer for checklist

bims/tasks/download_taxa_list.py

@@ -31,6 +31,7 @@ def __init__(self, get_data):
     total_taxa = taxa_list.count()
 
     tag_titles = []
+    additional_attributes_titles = []
 
     # Define the header update function
     def update_headers(_headers):
@@ -45,7 +46,11 @@ def update_headers(_headers):
             elif header == 'accepted_taxon':
                 header = ACCEPTED_TAXON
             header = header.replace('_or_', '/')
-            if not header.istitle() and header not in tag_titles:
+            if (
+                not header.istitle()
+                    and header not in tag_titles
+                    and header not in additional_attributes_titles
+            ):
                 header = header.replace('_', ' ').capitalize()
             if header == 'Sub species':
                 header = SUBSPECIES
@@ -58,6 +63,9 @@ def update_headers(_headers):
     sample_item = next(taxa_list.iterator())
     sample_serializer = TaxaCSVSerializer(sample_item)
     tag_titles = sample_serializer.context.get('tags', [])
+    additional_attributes_titles = sample_serializer.context.get(
+        'additional_attributes_titles', []
+    )
     headers = list(sample_serializer.data.keys())
     updated_headers = update_headers(headers)
 

bims/tests/test_checklist.py

@@ -1,14 +1,25 @@
 from django.test import TestCase
-from bims.models import BiologicalCollectionRecord, Dataset
-from bims.serializers.checklist_serializer import get_dataset_occurrences, ChecklistPDFSerializer, ChecklistSerializer
+
+from bims.api_views.checklist import checklist_collection_records
+from bims.models.dataset import Dataset
+from bims.models.download_request import DownloadRequest
+from bims.models.biological_collection_record import (
+    BiologicalCollectionRecord
+)
+from bims.serializers.checklist_serializer import (
+    get_dataset_occurrences, ChecklistPDFSerializer, ChecklistSerializer
+)
 from bims.tests.model_factories import (
     BiologicalCollectionRecordF,
     TaxonomyF,
     DatasetF,
     VernacularNameF,
-    SourceReferenceF
+    SourceReferenceF,
+    TaxonGroupF,
+    UserF
 )
 
+
 class TestGetDatasetOccurrences(TestCase):
 
     def setUp(self):
@@ -72,6 +83,12 @@ def setUp(self):
         self.context = {
             'collection_records': BiologicalCollectionRecord.objects.all()
         }
+        self.download_request = DownloadRequest.objects.create(
+            dashboard_url="http://example.com/dashboard",
+            resource_type="csv",
+            processing=True,
+            rejected=False
+        )
 
     def test_get_dataset_occurrences_with_dataset_keys(self):
         occurrences = BiologicalCollectionRecord.objects.filter(
@@ -128,3 +145,34 @@ def test_csv_serializer(self):
         self.assertTrue(Dataset.objects.filter(
             citation='Test Citation'
         ).exists())
+
+    def test_checklist_collection_records(self):
+        taxon_group = TaxonGroupF.create()
+        UserF.create(is_superuser=True)
+        BiologicalCollectionRecordF.create(
+            module_group=taxon_group,
+            data_type='private'
+        )
+        BiologicalCollectionRecordF.create(
+            module_group=taxon_group,
+            data_type='public'
+        )
+        BiologicalCollectionRecordF.create(
+            module_group=taxon_group,
+            data_type='sensitive'
+        )
+        dashboard_url = (
+            '/map/#site-detail/taxon=&search=&siteId=&collector=&category=&yearFrom=&yearTo=&months=&boundary=&userBoundary=&' +
+            f'referenceCategory=&spatialFilter=&reference=&endemic=&invasions=&conservationStatus=[]&modules={taxon_group.id}&validated=&' +
+            'sourceCollection=&module=occurrence&ecologicalCategory=&rank=&siteIdOpen=&orderBy=name&polygon=&dst=&ecosystemType='
+        )
+        download_request = DownloadRequest.objects.create(
+            resource_type='csv',
+            resource_name='checklist',
+            dashboard_url=dashboard_url
+        )
+        collection_records = checklist_collection_records(download_request)
+        self.assertEqual(
+            collection_records.count(),
+            3
+        )

bims/views/download_csv_taxa_list.py

@@ -190,13 +190,12 @@ class Meta:
             'cites_listing'
         )
 
-
-
     def __init__(self, *args, **kwargs):
         super(TaxaCSVSerializer, self).__init__(*args, **kwargs)
         self.context['headers'] = []
         self.context['additional_data'] = []
         self.context['tags'] = []
+        self.context['additional_attributes_titles'] = []
 
     def _ensure_headers(self, keys):
         if 'headers' not in self.context:
@@ -218,6 +217,9 @@ def _add_additional_attributes(self, instance, result):
                     continue
                 if attribute_name not in self.context['headers']:
                     self.context['headers'].append(attribute_name)
+                    self.context['additional_attributes_titles'].append(
+                        attribute_name
+                    )
                 if instance.additional_data:
                     result[attribute_name] = (
                         instance.additional_data.get(attribute_name, '')