From 51a9d1682eb24a02c2d78ce29d2352d40bc9922d Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Tue, 21 Dec 2021 19:13:36 +0100 Subject: [PATCH 01/14] Add single pixel photo to test insuree --- insuree/test_helpers.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/insuree/test_helpers.py b/insuree/test_helpers.py index 6bbbeef..a07bb10 100644 --- a/insuree/test_helpers.py +++ b/insuree/test_helpers.py @@ -37,6 +37,18 @@ def create_test_insuree(with_family=True, custom_props=None, family_custom_props return insuree +base64_blank_jpg = """ +/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL +/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAABAAEDASIAAhEBAxEB/8 +QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2Jyg +gkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLD +xMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ +3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eH +l6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD3+ +iiigD//2Q== +""" + + def create_test_photo(insuree_id, officer_id, custom_props=None): photo = InsureePhoto.objects.create( **{ @@ -47,6 +59,7 @@ def create_test_photo(insuree_id, officer_id, custom_props=None): "date": "2020-01-01", "validity_from": "2019-01-01", "audit_user_id": -1, + "photo": base64_blank_jpg, **(custom_props if custom_props else {}) } ) From 7e2d14591c9b387dd63318d43a3dcfedfe57daec Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Thu, 23 Jun 2022 16:12:22 +0200 Subject: [PATCH 02/14] OTC-612: GQL Handles photos from files --- insuree/apps.py | 13 +- insuree/gql_queries.py | 17 ++ insuree/schema.py | 1 - insuree/services.py | 42 ++-- insuree/tests/__init__.py | 3 + insuree/{ => tests}/test_graphql.py | 0 insuree/tests/test_insuree_photo.py | 180 ++++++++++++++++++ .../test_insuree_validation.py} | 0 8 files changed, 239 insertions(+), 17 deletions(-) create mode 100644 insuree/tests/__init__.py rename insuree/{ => tests}/test_graphql.py (100%) create mode 100644 insuree/tests/test_insuree_photo.py rename insuree/{tests.py => tests/test_insuree_validation.py} (100%) diff --git a/insuree/apps.py b/insuree/apps.py index a00b5d4..3e3a197 100644 --- a/insuree/apps.py +++ b/insuree/apps.py @@ -1,3 +1,5 @@ +import os + from django.apps import AppConfig from django.conf import settings @@ -63,7 +65,6 @@ def _configure_permissions(self, cfg): InsureeConfig.gql_mutation_create_insurees_perms = cfg["gql_mutation_create_insurees_perms"] InsureeConfig.gql_mutation_update_insurees_perms = cfg["gql_mutation_update_insurees_perms"] InsureeConfig.gql_mutation_delete_insurees_perms = cfg["gql_mutation_delete_insurees_perms"] - InsureeConfig.insuree_photos_root_path = cfg["insuree_photos_root_path"] InsureeConfig.insuree_number_validator = cfg["insuree_number_validator"] InsureeConfig.insuree_number_length = cfg["insuree_number_length"] InsureeConfig.insuree_number_modulo_root = cfg["insuree_number_modulo_root"] @@ -81,6 +82,7 @@ def ready(self): self._configure_permissions(cfg) self._configure_fake_insurees(cfg) self._configure_renewal(cfg) + self._configure_photo_root(cfg) # Getting these at runtime for easier testing @classmethod @@ -106,3 +108,12 @@ def set_dataloaders(self, dataloaders): @classmethod def __get_from_settings_or_default(cls, attribute_name, default=None): return getattr(settings, attribute_name) if hasattr(settings, attribute_name) else default + + def _configure_photo_root(self, cfg): + # TODO: To be confirmed. I left loading from config for integrity reasons + # but it could be based on env variable only. + # Also we could determine global file root for all stored files across modules. + if from_config := cfg.get("insuree_photos_root_path", None): + InsureeConfig.insuree_photos_root_path = from_config + elif from_env := os.getenv("PHOTO_ROOT_PATH", None): + InsureeConfig.insuree_photos_root_path = from_env diff --git a/insuree/gql_queries.py b/insuree/gql_queries.py index 03b7d36..071a6a3 100644 --- a/insuree/gql_queries.py +++ b/insuree/gql_queries.py @@ -1,11 +1,15 @@ import graphene from graphene_django import DjangoObjectType + +from .apps import InsureeConfig from .models import Insuree, InsureePhoto, Education, Profession, Gender, IdentificationType, \ Family, FamilyType, ConfirmationType, Relation, InsureePolicy, FamilyMutation, InsureeMutation from location.schema import LocationGQLType from policy.gql_queries import PolicyGQLType from core import prefix_filterset, filter_validity, ExtendedConnection +from .services import load_photo_file + class GenderGQLType(DjangoObjectType): class Meta: @@ -16,6 +20,15 @@ class Meta: class PhotoGQLType(DjangoObjectType): + photo = graphene.String() + + def resolve_photo(self, info): + if self.photo: + return self.photo + elif InsureeConfig.insuree_photos_root_path and self.folder and self.filename: + return load_photo_file(self.folder, self.filename) + return None + class Meta: model = InsureePhoto filter_fields = { @@ -76,6 +89,7 @@ class Meta: class InsureeGQLType(DjangoObjectType): age = graphene.Int(source='age') client_mutation_id = graphene.String() + photo = PhotoGQLType() def resolve_current_village(self, info): if "location_loader" in info.context.dataloaders and self.current_village_id: @@ -96,6 +110,9 @@ def resolve_health_facility(self, info): ) return self.health_facility + def resolve_photo(self, info): + return self.photo + class Meta: model = Insuree filter_fields = { diff --git a/insuree/schema.py b/insuree/schema.py index ed50844..1e3264a 100644 --- a/insuree/schema.py +++ b/insuree/schema.py @@ -151,7 +151,6 @@ def resolve_insurees(self, info, **kwargs): (Q(current_village__isnull=True) & Q(**{family_location: parent_location}))] return gql_optimizer.query(Insuree.objects.filter(*filters).all(), info) - def resolve_family_members(self, info, **kwargs): if not info.context.user.has_perms(InsureeConfig.gql_query_insurees_perms): raise PermissionDenied(_("unauthorized")) diff --git a/insuree/services.py b/insuree/services.py index 389bba3..968838a 100644 --- a/insuree/services.py +++ b/insuree/services.py @@ -2,6 +2,7 @@ import logging import pathlib import uuid +from os import path from core.apps import CoreConfig from django.db.models import Q @@ -114,7 +115,8 @@ def handle_insuree_photo(user, now, insuree, data): data['validity_from'] = now data['insuree_id'] = insuree.id photo_bin = data.get('photo', None) - if photo_bin and InsureeConfig.insuree_photos_root_path and photo_bin != insuree_photo.photo: + if photo_bin and InsureeConfig.insuree_photos_root_path \ + and (insuree_photo is None or insuree_photo.photo != photo_bin): (file_dir, file_name) = create_file(now, insuree.id, photo_bin) data.pop('photo', None) data['folder'] = file_dir @@ -143,24 +145,34 @@ def photo_changed(insuree_photo, data): (data and insuree_photo and insuree_photo.photo != data.get('photo', None)) -def create_file(date, insuree_id, photo_bin): - date_iso = date.isoformat() +def _photo_dir(file_dir, file_name): root = InsureeConfig.insuree_photos_root_path - file_dir = '%s/%s/%s/%s' % ( - date_iso[0:4], - date_iso[5:7], - date_iso[8:10], - insuree_id - ) - file_name = uuid.uuid4() - file_path = '%s/%s' % (file_dir, file_name) - pathlib.Path('%s/%s' % (root, file_dir)).mkdir(parents=True, exist_ok=True) - f = open('%s/%s' % (root, file_path), "xb") - f.write(base64.b64decode(photo_bin)) - f.close() + return path.join(root, file_dir, file_name) + + +def _create_dir(file_dir): + root = InsureeConfig.insuree_photos_root_path + pathlib.Path(path.join(root, file_dir))\ + .mkdir(parents=True, exist_ok=True) + + +def create_file(date, insuree_id, photo_bin): + file_dir = path.join(str(date.year), str(date.month), str(date.day), str(insuree_id)) + file_name = str(uuid.uuid4()) + + _create_dir(file_dir) + with open(_photo_dir(file_dir, file_name), "xb") as f: + f.write(base64.b64decode(photo_bin)) + f.close() return file_dir, file_name +def load_photo_file(file_dir, file_name): + photo_path = _photo_dir(file_dir, file_name) + with open(photo_path, "rb") as f: + return base64.b64encode(f.read()).decode("utf-8") + + class InsureeService: def __init__(self, user): self.user = user diff --git a/insuree/tests/__init__.py b/insuree/tests/__init__.py new file mode 100644 index 0000000..bfe6ead --- /dev/null +++ b/insuree/tests/__init__.py @@ -0,0 +1,3 @@ +from .test_graphql import InsureeGQLTestCase +from .test_insuree_photo import InsureePhotoTest +from .test_insuree_validation import InsureeValidationTest diff --git a/insuree/test_graphql.py b/insuree/tests/test_graphql.py similarity index 100% rename from insuree/test_graphql.py rename to insuree/tests/test_graphql.py diff --git a/insuree/tests/test_insuree_photo.py b/insuree/tests/test_insuree_photo.py new file mode 100644 index 0000000..a34f57c --- /dev/null +++ b/insuree/tests/test_insuree_photo.py @@ -0,0 +1,180 @@ +import uuid +from unittest import mock +from unittest.mock import PropertyMock + +from django.test import TestCase +from core.forms import User + +from graphene import Schema +from graphene.test import Client +from insuree import schema as insuree_schema +from insuree.models import Insuree +from insuree.test_helpers import create_test_insuree +from core.services import create_or_update_interactive_user, create_or_update_core_user + +from insuree.services import validate_insuree_number +from unittest.mock import ANY +from django.conf import settings + + +class InsureePhotoTest(TestCase): + _TEST_USER_NAME = "TestUserTest2" + _TEST_USER_PASSWORD = "TestPasswordTest2" + _TEST_DATA_USER = { + "username": _TEST_USER_NAME, + "last_name": _TEST_USER_NAME, + "password": _TEST_USER_PASSWORD, + "other_names": _TEST_USER_NAME, + "user_types": "INTERACTIVE", + "language": "en", + "roles": [4], + } + photo_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAcAAAAHCAYAAADEUlfTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAOxAAADsQBlSsOGwAAABl0RVh0U29mdHdhcmUAd3d3Lmlua3NjYXBlLm9yZ5vuPBoAAAB7SURBVAiZLc0xDsIwEETRP+t1wpGoaDg6DUgpEAUNNyH2DkXon/S03W+uKiIaANmS07Jim2ytc75cAWMbAJF8Xg9iycQV1AywALCh9yTWtXN4Yx9Agu++EyAkA0IxQQcdc5BjDCJEGST9T3AZvZ+bXUYhMhtzFlWmZvEDKAM9L8CDZ0EAAAAASUVORK5CYII=" + test_photo_path, test_photo_uuid = "some/file/path", str(uuid.uuid4()) + + class BaseTestContext: + def __init__(self, user): + self.user = user + + def setUp(self): + super(InsureePhotoTest, self).setUp() + self._TEST_USER = self._get_or_create_user_api() + self.insuree = create_test_insuree(custom_props={'chf_id': '110707070'}) + self.row_sec = settings.ROW_SECURITY + settings.ROW_SECURITY = False + + def tearDown(self) -> None: + settings.ROW_SECURITY = self.row_sec + + @classmethod + def setUpClass(cls): + # Signals are not automatically bound in unit tests + super(InsureePhotoTest, cls).setUpClass() + cls.insuree_client = Client(Schema(mutation=insuree_schema.Mutation, query=insuree_schema.Query)) + insuree_schema.bind_signals() + + def test_add_photo_save_db(self): + self.__call_photo_mutation() + self.assertEqual(self.insuree.photo.photo, self.photo_base64) + + def test_pull_photo_db(self): + self.__call_photo_mutation() + query_result = self.__call_photo_query() + try: + gql_photo = query_result['data']['insurees']['edges'][0]['node']['photo'] + self.assertEqual(gql_photo['photo'], self.photo_base64) + except Exception as e: + print(query_result) + raise e + + @mock.patch('insuree.services.InsureeConfig') + @mock.patch('insuree.services.create_file') + def test_add_photo_save_files(self, create_file, insuree_config): + create_file.return_value = self.test_photo_path, self.test_photo_uuid + insuree_config.insuree_photos_root_path = PropertyMock(return_value="insuree_root_path") + + self.__call_photo_mutation() + + self.assertEqual(self.insuree.photo.folder, "some/file/path") + self.assertEqual(self.insuree.photo.filename, str(self.test_photo_uuid)) + create_file.assert_called_once_with(ANY, self.insuree.id, self.photo_base64) + + @mock.patch('insuree.services.InsureeConfig') + @mock.patch('insuree.gql_queries.InsureeConfig') + @mock.patch('insuree.services.create_file') + @mock.patch('insuree.gql_queries.load_photo_file') + def test_pull_photo_file_path(self, load_photo_file, create_file, insuree_config, insuree_config2): + load_photo_file.return_value = self.photo_base64 + create_file.return_value = self.test_photo_path, self.test_photo_uuid + insuree_config.insuree_photos_root_path = PropertyMock(return_value="insuree_root_path") + insuree_config2.insuree_photos_root_path = PropertyMock(return_value="insuree_root_path") + self.__call_photo_mutation() + query_result = self.__call_photo_query() + gql_photo = query_result['data']['insurees']['edges'][0]['node']['photo'] + self.assertEqual(gql_photo['photo'], self.photo_base64) + load_photo_file.assert_called_once_with(self.test_photo_path, str(self.test_photo_uuid)) + + def __call_photo_mutation(self): + mutation = self.__update_photo_mutation(self.insuree, self._TEST_USER) + context = self.BaseTestContext(self._TEST_USER) + self.insuree_client.execute(mutation, context=context) + self.insuree.refresh_from_db() + + def __call_photo_query(self): + query = self.__get_insuree_query(self.insuree) + context = self.BaseTestContext(self._TEST_USER) + return self.insuree_client.execute(query, context=context) + + def __update_photo_mutation(self, insuree: Insuree, officer: User): + return F''' + mutation + {{ + updateInsuree(input: {{ + clientMutationId: "c9598c58-26c8-47d7-b33e-c8d606eb9ab3" + clientMutationLabel: "Update insuree - {insuree.chf_id}" + uuid: "{insuree.uuid}" + chfId: "{insuree.chf_id}" + lastName: "{insuree.last_name}" + otherNames: "{insuree.other_names}" + genderId: "M" + dob: "1950-07-12" + head: true + marital: "M" + photo:{{ + uuid: "C194B52F-117F-4B43-B143-26C5F0A45153" + officerId: {officer._u.id} + date: "2022-06-21" + photo: "{self.photo_base64}" + }} + cardIssued:false + familyId: {insuree.family.id} + }}) + {{ + internalId + clientMutationId + }} + }} + ''' + + def _get_or_create_user_api(self): + try: + return User.objects.filter(username=self._TEST_USER_NAME).get() + except User.DoesNotExist: + return self.__create_user_interactive_core() + + def __create_user_interactive_core(self): + data = self._TEST_DATA_USER + i_user, i_user_created = create_or_update_interactive_user( + user_id=None, data=data, audit_user_id=999, connected=False + ) + create_or_update_core_user(user_uuid=None, username=self._TEST_USER_NAME, i_user=i_user) + return User.objects.filter(username=self._TEST_USER_NAME).get() + + def __get_insuree_query(self, insuree): + return F''' +{{ + insurees(uuid: "{insuree.uuid}") {{ + pageInfo {{ + hasNextPage, + hasPreviousPage, + startCursor, + endCursor + }} + edges {{ + node {{ + uuid, + chfId, + photo {{ + id, + uuid, + date, + folder, + filename, + officerId, + photo + }} + }} + }} + }} +}} +''' diff --git a/insuree/tests.py b/insuree/tests/test_insuree_validation.py similarity index 100% rename from insuree/tests.py rename to insuree/tests/test_insuree_validation.py From b1129fbac0605a03113aba95b3010460d81ac724 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Thu, 23 Jun 2022 16:15:01 +0200 Subject: [PATCH 03/14] OTC-612: Print debug log on failed unit tests in CI --- .github/workflows/openmis-module-test.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/openmis-module-test.yml b/.github/workflows/openmis-module-test.yml index d103348..d39accc 100644 --- a/.github/workflows/openmis-module-test.yml +++ b/.github/workflows/openmis-module-test.yml @@ -77,6 +77,7 @@ jobs: # black --check . - name: Django tests + id: django_tests working-directory: ./openimis/openIMIS run: | export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')" @@ -85,9 +86,11 @@ jobs: python manage.py migrate python init_test_db.py | grep . | uniq -c python manage.py test --keepdb $MODULE_NAME + if [ $? == 1 ]; then cat debug.log; fi env: SECRET_KEY: secret DEBUG: true + DJANGO_LOG_LEVEL: DEBUG #DJANGO_SETTINGS_MODULE: hat.settings DB_HOST: localhost DB_PORT: 1433 @@ -96,5 +99,8 @@ jobs: DB_PASSWORD: GitHub999 #DEV_SERVER: true SITE_ROOT: api - - + - name: Print debug logs on failed tests + if: failure() + working-directory: ./openimis/openIMIS + run: | + cat debug.log From db902f90e6abaafd7ea35cd8e463a258aac00b8a Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Thu, 21 Jul 2022 10:00:20 +0200 Subject: [PATCH 04/14] OMT-301 Gender should be mandatory --- insuree/gql_mutations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/insuree/gql_mutations.py b/insuree/gql_mutations.py index 35fe1a9..d858344 100644 --- a/insuree/gql_mutations.py +++ b/insuree/gql_mutations.py @@ -33,7 +33,7 @@ class InsureeBase: chf_id = graphene.String(max_length=12, required=False) last_name = graphene.String(max_length=100, required=True) other_names = graphene.String(max_length=100, required=True) - gender_id = graphene.String(max_length=1, required=False) + gender_id = graphene.String(max_length=1, required=True, description="Was mandatory in Legacy but not in modular") dob = graphene.Date(required=True) head = graphene.Boolean(required=False) marital = graphene.String(max_length=1, required=False) From f4171727013c2fff2b3746085c63d6f01226de64 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Tue, 26 Jul 2022 10:42:12 +0200 Subject: [PATCH 05/14] OP-818: Fixed Insuree Number validation --- insuree/gql_mutations.py | 4 ++-- insuree/schema.py | 5 +++-- insuree/services.py | 6 +++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/insuree/gql_mutations.py b/insuree/gql_mutations.py index 35fe1a9..beff8f9 100644 --- a/insuree/gql_mutations.py +++ b/insuree/gql_mutations.py @@ -152,7 +152,7 @@ def async_mutate(cls, user, **data): data['validity_from'] = TimeUtils.now() client_mutation_id = data.get("client_mutation_id") # Validate insuree number right away - errors = validate_insuree_number(data.get("head_insuree", {}).get("chf_id", None)) + errors = validate_insuree_number(data.get("head_insuree", {}).get("chf_id", None), True) if errors: return errors family = update_or_create_family(data, user) @@ -253,7 +253,7 @@ def async_mutate(cls, user, **data): data['validity_from'] = TimeUtils.now() client_mutation_id = data.get("client_mutation_id") # Validate insuree number right away - errors = validate_insuree_number(data.get("chf_id", None)) + errors = validate_insuree_number(data.get("chf_id", None), True) if errors: return errors insuree = update_or_create_insuree(data, user) diff --git a/insuree/schema.py b/insuree/schema.py index 1e3264a..67e81f4 100644 --- a/insuree/schema.py +++ b/insuree/schema.py @@ -93,11 +93,12 @@ class Query(graphene.ObjectType): insuree_number_validity = graphene.Field( graphene.Boolean, insuree_number=graphene.String(required=True), + new_insuree=graphene.Boolean(required=False), description="Checks that the specified insuree number is valid" ) - def resolve_insuree_number_validity(self, info, insuree_number=None): - errors = validate_insuree_number(insuree_number) + def resolve_insuree_number_validity(self, info, **kwargs): + errors = validate_insuree_number(kwargs['insuree_number'], kwargs['new_insuree']) if errors: return False else: diff --git a/insuree/services.py b/insuree/services.py index 968838a..168c370 100644 --- a/insuree/services.py +++ b/insuree/services.py @@ -40,7 +40,11 @@ def create_insuree_renewal_detail(policy_renewal): photo.insuree_id, detail.id, detail_created) -def validate_insuree_number(insuree_number): +def validate_insuree_number(insuree_number, is_new_insuree=False): + if is_new_insuree: + if Insuree.objects.filter(chf_id=insuree_number).exists(): + return [{"message": "Insuree number has to be unique, %s exists in system" % insuree_number}] + if InsureeConfig.get_insuree_number_validator(): return InsureeConfig.get_insuree_number_validator()(insuree_number) if InsureeConfig.get_insuree_number_length(): From acb290493e4bcb1c96a0733a51bfd7a9114826a9 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Tue, 26 Jul 2022 12:44:19 +0200 Subject: [PATCH 06/14] OP-818: Fixed unit test for insuree number validation --- insuree/tests/test_graphql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/insuree/tests/test_graphql.py b/insuree/tests/test_graphql.py index a0411b0..d3f7ff1 100644 --- a/insuree/tests/test_graphql.py +++ b/insuree/tests/test_graphql.py @@ -43,7 +43,7 @@ def test_query_insuree_number_validity(self): response = self.query( ''' { - insureeNumberValidity(insureeNumber:"123456782") + insureeNumberValidity(insureeNumber:"123456782", newInsuree:false) } ''', headers={"HTTP_AUTHORIZATION": f"Bearer {self.admin_token}"}, From 729e1fc2664257da0f8ea5bd635174fe9fb18211 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Thu, 21 Jul 2022 10:00:20 +0200 Subject: [PATCH 07/14] OMT-301 Gender should be mandatory --- insuree/gql_mutations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/insuree/gql_mutations.py b/insuree/gql_mutations.py index beff8f9..23620e3 100644 --- a/insuree/gql_mutations.py +++ b/insuree/gql_mutations.py @@ -33,7 +33,7 @@ class InsureeBase: chf_id = graphene.String(max_length=12, required=False) last_name = graphene.String(max_length=100, required=True) other_names = graphene.String(max_length=100, required=True) - gender_id = graphene.String(max_length=1, required=False) + gender_id = graphene.String(max_length=1, required=True, description="Was mandatory in Legacy but not in modular") dob = graphene.Date(required=True) head = graphene.Boolean(required=False) marital = graphene.String(max_length=1, required=False) From deca69f4be9888739f40552ecd28734510b8c907 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Mon, 1 Aug 2022 10:54:37 +0200 Subject: [PATCH 08/14] PostgreSQL is case sensitive when mixing case The database has InsureeID. It doesn't matter for SQL Server but Postgres requires double quotes when there are uppercase characters and then becomes case sensitive. --- insuree/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/insuree/models.py b/insuree/models.py index b6c3870..e51145e 100644 --- a/insuree/models.py +++ b/insuree/models.py @@ -297,7 +297,7 @@ class Meta: class InsureePolicy(core_models.VersionedModel): id = models.AutoField(db_column='InsureePolicyID', primary_key=True) - insuree = models.ForeignKey(Insuree, models.DO_NOTHING, db_column='InsureeId', related_name="insuree_policies") + insuree = models.ForeignKey(Insuree, models.DO_NOTHING, db_column='InsureeID', related_name="insuree_policies") policy = models.ForeignKey("policy.Policy", models.DO_NOTHING, db_column='PolicyId', related_name="insuree_policies") From bf4e7936ba8cec1cd22866d9254e597ebd3d8aac Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Mon, 1 Aug 2022 10:56:20 +0200 Subject: [PATCH 09/14] Add picture copy from unopened file This is necessary for the offline XML import --- insuree/services.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/insuree/services.py b/insuree/services.py index 168c370..a83671b 100644 --- a/insuree/services.py +++ b/insuree/services.py @@ -1,6 +1,7 @@ import base64 import logging import pathlib +import shutil import uuid from os import path @@ -171,6 +172,15 @@ def create_file(date, insuree_id, photo_bin): return file_dir, file_name +def copy_file(date, insuree_id, original_file): + file_dir = path.join(str(date.year), str(date.month), str(date.day), str(insuree_id)) + file_name = str(uuid.uuid4()) + + _create_dir(file_dir) + shutil.copy2(original_file, _photo_dir(file_dir, file_name)) + return file_dir, file_name + + def load_photo_file(file_dir, file_name): photo_path = _photo_dir(file_dir, file_name) with open(photo_path, "rb") as f: From 6791ef5522f050bae77697c061d7947d2c402024 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Tue, 9 Aug 2022 23:12:43 +0200 Subject: [PATCH 10/14] Adapt migrations for PostgreSQL --- insuree/migrations/0007_auto_20200722_0940.py | 9 ++++++-- insuree/migrations/0008_auto_20200731_0443.py | 4 +++- insuree/migrations/0010_auto_20200731_0524.py | 4 ++-- insuree/migrations/0011_auto_20200807_1309.py | 21 +++++++++++++++--- insuree/migrations/0013_auto_20211103_1023.py | 22 ++++++++++++------- 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/insuree/migrations/0007_auto_20200722_0940.py b/insuree/migrations/0007_auto_20200722_0940.py index 5ccaff2..cd0b9bd 100644 --- a/insuree/migrations/0007_auto_20200722_0940.py +++ b/insuree/migrations/0007_auto_20200722_0940.py @@ -1,4 +1,5 @@ # Generated by Django 3.0.3 on 2020-07-22 09:40 +from django.conf import settings import core.fields import datetime @@ -23,8 +24,12 @@ class Migration(migrations.Migration): # ), # so let's make it raw SQL... migrations.RunSQL( - "ALTER TABLE[tblInsuree] ADD CONSTRAINT[tblInsuree_CurrentVillage_8ea25085_fk_tblLocations_LocationId] \ - FOREIGN KEY([CurrentVillage]) REFERENCES[tblLocations]([LocationId]);" + "ALTER TABLE [tblInsuree] ADD CONSTRAINT " + "[tblInsuree_CurrentVillage_8ea25085_fk_tblLocations_LocationId] " + "FOREIGN KEY([CurrentVillage]) REFERENCES[tblLocations]([LocationId]);" + if "sql_server" in settings.DB_ENGINE else + 'ALTER TABLE "tblInsuree" ADD CONSTRAINT "tblInsuree_CurrentVillage_8ea25085_fk_tblLocations_LocationId" ' + ' FOREIGN KEY("CurrentVillage") REFERENCES "tblLocations"("LocationId");' ) ] diff --git a/insuree/migrations/0008_auto_20200731_0443.py b/insuree/migrations/0008_auto_20200731_0443.py index fefa60e..66085d6 100644 --- a/insuree/migrations/0008_auto_20200731_0443.py +++ b/insuree/migrations/0008_auto_20200731_0443.py @@ -1,5 +1,5 @@ # Generated by Django 3.0.3 on 2020-07-31 04:43 - +from django.conf import settings from django.db import migrations @@ -12,5 +12,7 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL( "ALTER TABLE[tblInsuree] ALTER COLUMN FamilyID [int] NULL;" + if "sql_server" in settings.DB_ENGINE else + 'ALTER TABLE "tblInsuree" ALTER COLUMN "FamilyID" DROP NOT NULL;' ) ] diff --git a/insuree/migrations/0010_auto_20200731_0524.py b/insuree/migrations/0010_auto_20200731_0524.py index 4afc722..de8e30f 100644 --- a/insuree/migrations/0010_auto_20200731_0524.py +++ b/insuree/migrations/0010_auto_20200731_0524.py @@ -10,6 +10,6 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunSQL('ALTER TABLE tblInsuree ADD JsonExt TEXT'), - migrations.RunSQL('ALTER TABLE tblFamilies ADD JsonExt TEXT'), + migrations.RunSQL('ALTER TABLE "tblInsuree" ADD "JsonExt" TEXT'), + migrations.RunSQL('ALTER TABLE "tblFamilies" ADD "JsonExt" TEXT'), ] diff --git a/insuree/migrations/0011_auto_20200807_1309.py b/insuree/migrations/0011_auto_20200807_1309.py index 7ee50d8..72ba40e 100644 --- a/insuree/migrations/0011_auto_20200807_1309.py +++ b/insuree/migrations/0011_auto_20200807_1309.py @@ -1,4 +1,5 @@ # Generated by Django 3.0.3 on 2020-08-07 13:09 +from django.conf import settings import core.fields from django.db import migrations, models @@ -12,9 +13,23 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunSQL('ALTER TABLE tblPhotos ADD LegacyID [int] NULL, photo [TEXT] null'), - migrations.RunSQL('ALTER TABLE tblPhotos ALTER COLUMN PhotoFolder nvarchar(255) NULL'), - migrations.RunSQL('ALTER TABLE tblPhotos ALTER COLUMN PhotoFileName nvarchar(255) NULL'), + migrations.RunSQL( + 'ALTER TABLE "tblPhotos" ADD "LegacyID" int NULL, photo TEXT null' + if "sql_server" in settings.DB_ENGINE else + 'ALTER TABLE "tblPhotos" ADD "LegacyID" int; ALTER TABLE "tblPhotos" add photo TEXT;' + ), + migrations.RunSQL( + 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFolder" nvarchar(255) NULL' + if "sql_server" in settings.DB_ENGINE else + 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFolder" TYPE VARCHAR(255); ' + 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFolder" DROP NOT NULL;' + ), + migrations.RunSQL( + 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFileName" nvarchar(255) NULL' + if "sql_server" in settings.DB_ENGINE else + 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFileName" TYPE VARCHAR(255);' + 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFileName" DROP NOT NULL;' + ), migrations.CreateModel( name='InsureePhoto', fields=[ diff --git a/insuree/migrations/0013_auto_20211103_1023.py b/insuree/migrations/0013_auto_20211103_1023.py index fcc3c2e..3cd91e7 100644 --- a/insuree/migrations/0013_auto_20211103_1023.py +++ b/insuree/migrations/0013_auto_20211103_1023.py @@ -1,5 +1,5 @@ # Generated by Django 3.0.14 on 2021-11-03 10:23 - +from django.conf import settings from django.db import migrations @@ -12,12 +12,18 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL( """ - CREATE NONCLUSTERED INDEX ix_tblInsuree_validity ON [dbo].[tblInsuree] - ( - [ValidityFrom] ASC, - [LegacyID] ASC, - InsureeId - )""", - reverse_sql="DROP index ix_tblInsuree_validity on tblInsuree", + CREATE NONCLUSTERED INDEX ix_tblInsuree_validity ON [dbo].[tblInsuree] + ( + [ValidityFrom] ASC, + [LegacyID] ASC, + InsureeID + )""" if "sql_server" in settings.DB_ENGINE else """ + CREATE INDEX "ix_tblInsuree_validity" ON "tblInsuree" + ( + "ValidityFrom" ASC, + "LegacyID" ASC, + "InsureeID" + )""", + reverse_sql='DROP index "ix_tblInsuree_validity" on "tblInsuree"', ) ] From 1a8c0e847bde0c0ab111d6801671aae59e8beb5c Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Wed, 10 Aug 2022 23:58:41 +0200 Subject: [PATCH 11/14] Add PostgreSQL json_ext --- insuree/migrations/0010_auto_20200731_0524.py | 10 +++++++--- insuree/models.py | 3 +++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/insuree/migrations/0010_auto_20200731_0524.py b/insuree/migrations/0010_auto_20200731_0524.py index de8e30f..db3f98f 100644 --- a/insuree/migrations/0010_auto_20200731_0524.py +++ b/insuree/migrations/0010_auto_20200731_0524.py @@ -1,5 +1,5 @@ # Generated by Django 3.0.3 on 2020-07-31 05:24 - +from django.conf import settings from django.db import migrations @@ -10,6 +10,10 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunSQL('ALTER TABLE "tblInsuree" ADD "JsonExt" TEXT'), - migrations.RunSQL('ALTER TABLE "tblFamilies" ADD "JsonExt" TEXT'), + migrations.RunSQL('ALTER TABLE [tblInsuree] ADD [JsonExt] TEXT' + if "sql_server" in settings.DB_ENGINE else + 'ALTER TABLE "tblInsuree" ADD "JsonExt" jsonb'), + migrations.RunSQL('ALTER TABLE [tblFamilies] ADD [JsonExt] TEXT' + if "sql_server" in settings.DB_ENGINE else + 'ALTER TABLE "tblFamilies" ADD "JsonExt" jsonb'), ] diff --git a/insuree/models.py b/insuree/models.py index e51145e..d81214c 100644 --- a/insuree/models.py +++ b/insuree/models.py @@ -1,5 +1,7 @@ import uuid +from jsonfallback.fields import FallbackJSONField + import core from core import models as core_models from django.conf import settings @@ -249,6 +251,7 @@ def is_adult(self, reference_date=None): health_facility = models.ForeignKey( location_models.HealthFacility, models.DO_NOTHING, db_column='HFID', blank=True, null=True, related_name='insurees') + json_ext = FallbackJSONField(db_column="JsonExt", blank=True, null=True) offline = models.BooleanField(db_column='isOffline', blank=True, null=True) audit_user_id = models.IntegerField(db_column='AuditUserID') From 6a5d4b50ff7e487b8bfea0753471f4d6687cf584 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Wed, 10 Aug 2022 23:58:57 +0200 Subject: [PATCH 12/14] Fix missing security on insuree family overview --- insuree/reports/insuree_family_overview.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/insuree/reports/insuree_family_overview.py b/insuree/reports/insuree_family_overview.py index 87ef6b7..b590bd3 100644 --- a/insuree/reports/insuree_family_overview.py +++ b/insuree/reports/insuree_family_overview.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.db.models import Q, F # If manually pasting from reportbro and you have test data, search and replace \" with \\" @@ -1709,16 +1710,17 @@ def insuree_family_overview_query(user, date_from=None, date_to=None, **kwargs): if date_to: filters &= Q(validity_from__lte=date_to + datetimedelta(days=1)) - # TODO use auth from Quentin's PR - # if settings.ROW_SECURITY: - # from location.models import UserDistrict - # dist = UserDistrict.get_user_districts(user._u) - # queryset = queryset.filter( - # health_facility__location__id__in=[l.location_id for l in dist] - # ) + if settings.ROW_SECURITY: + from location.models import UserDistrict + dist = UserDistrict.get_user_districts(user._u) + queryset = Insuree.objects.filter( + health_facility__location__id__in=[l.location_id for l in dist] + ) + else: + queryset = Insuree.objects queryset = ( - Insuree.objects.filter(filters) + queryset.filter(filters) .values( "chf_id", "other_names", From 31c6a647d24cb1812787111a9d3601bf36bf368b Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Thu, 15 Sep 2022 15:01:15 +0200 Subject: [PATCH 13/14] Django 3.1+ upgrade --- insuree/migrations/0007_auto_20200722_0940.py | 2 +- insuree/migrations/0008_auto_20200731_0443.py | 2 +- insuree/migrations/0010_auto_20200731_0524.py | 4 ++-- insuree/migrations/0011_auto_20200807_1309.py | 6 +++--- insuree/migrations/0013_auto_20211103_1023.py | 2 +- insuree/models.py | 8 ++------ insuree/schema.py | 2 +- 7 files changed, 11 insertions(+), 15 deletions(-) diff --git a/insuree/migrations/0007_auto_20200722_0940.py b/insuree/migrations/0007_auto_20200722_0940.py index cd0b9bd..af4fcbf 100644 --- a/insuree/migrations/0007_auto_20200722_0940.py +++ b/insuree/migrations/0007_auto_20200722_0940.py @@ -27,7 +27,7 @@ class Migration(migrations.Migration): "ALTER TABLE [tblInsuree] ADD CONSTRAINT " "[tblInsuree_CurrentVillage_8ea25085_fk_tblLocations_LocationId] " "FOREIGN KEY([CurrentVillage]) REFERENCES[tblLocations]([LocationId]);" - if "sql_server" in settings.DB_ENGINE else + if settings.MSSQL else 'ALTER TABLE "tblInsuree" ADD CONSTRAINT "tblInsuree_CurrentVillage_8ea25085_fk_tblLocations_LocationId" ' ' FOREIGN KEY("CurrentVillage") REFERENCES "tblLocations"("LocationId");' ) diff --git a/insuree/migrations/0008_auto_20200731_0443.py b/insuree/migrations/0008_auto_20200731_0443.py index 66085d6..d9b474e 100644 --- a/insuree/migrations/0008_auto_20200731_0443.py +++ b/insuree/migrations/0008_auto_20200731_0443.py @@ -12,7 +12,7 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL( "ALTER TABLE[tblInsuree] ALTER COLUMN FamilyID [int] NULL;" - if "sql_server" in settings.DB_ENGINE else + if settings.MSSQL else 'ALTER TABLE "tblInsuree" ALTER COLUMN "FamilyID" DROP NOT NULL;' ) ] diff --git a/insuree/migrations/0010_auto_20200731_0524.py b/insuree/migrations/0010_auto_20200731_0524.py index db3f98f..a784d3f 100644 --- a/insuree/migrations/0010_auto_20200731_0524.py +++ b/insuree/migrations/0010_auto_20200731_0524.py @@ -11,9 +11,9 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL('ALTER TABLE [tblInsuree] ADD [JsonExt] TEXT' - if "sql_server" in settings.DB_ENGINE else + if settings.MSSQL else 'ALTER TABLE "tblInsuree" ADD "JsonExt" jsonb'), migrations.RunSQL('ALTER TABLE [tblFamilies] ADD [JsonExt] TEXT' - if "sql_server" in settings.DB_ENGINE else + if settings.MSSQL else 'ALTER TABLE "tblFamilies" ADD "JsonExt" jsonb'), ] diff --git a/insuree/migrations/0011_auto_20200807_1309.py b/insuree/migrations/0011_auto_20200807_1309.py index 72ba40e..2085340 100644 --- a/insuree/migrations/0011_auto_20200807_1309.py +++ b/insuree/migrations/0011_auto_20200807_1309.py @@ -15,18 +15,18 @@ class Migration(migrations.Migration): operations = [ migrations.RunSQL( 'ALTER TABLE "tblPhotos" ADD "LegacyID" int NULL, photo TEXT null' - if "sql_server" in settings.DB_ENGINE else + if settings.MSSQL else 'ALTER TABLE "tblPhotos" ADD "LegacyID" int; ALTER TABLE "tblPhotos" add photo TEXT;' ), migrations.RunSQL( 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFolder" nvarchar(255) NULL' - if "sql_server" in settings.DB_ENGINE else + if settings.MSSQL else 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFolder" TYPE VARCHAR(255); ' 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFolder" DROP NOT NULL;' ), migrations.RunSQL( 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFileName" nvarchar(255) NULL' - if "sql_server" in settings.DB_ENGINE else + if settings.MSSQL else 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFileName" TYPE VARCHAR(255);' 'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFileName" DROP NOT NULL;' ), diff --git a/insuree/migrations/0013_auto_20211103_1023.py b/insuree/migrations/0013_auto_20211103_1023.py index 3cd91e7..d9bbd70 100644 --- a/insuree/migrations/0013_auto_20211103_1023.py +++ b/insuree/migrations/0013_auto_20211103_1023.py @@ -17,7 +17,7 @@ class Migration(migrations.Migration): [ValidityFrom] ASC, [LegacyID] ASC, InsureeID - )""" if "sql_server" in settings.DB_ENGINE else """ + )""" if settings.MSSQL else """ CREATE INDEX "ix_tblInsuree_validity" ON "tblInsuree" ( "ValidityFrom" ASC, diff --git a/insuree/models.py b/insuree/models.py index d81214c..d292bed 100644 --- a/insuree/models.py +++ b/insuree/models.py @@ -1,7 +1,5 @@ import uuid -from jsonfallback.fields import FallbackJSONField - import core from core import models as core_models from django.conf import settings @@ -91,14 +89,13 @@ class Family(core_models.VersionedModel, core_models.ExtendableModel): location = models.ForeignKey( location_models.Location, models.DO_NOTHING, db_column='LocationId', blank=True, null=True) - # Need to be NullBooleanField (BooleanField + null=True is not enough) for Graphene to map properly - poverty = models.NullBooleanField(db_column='Poverty', blank=True, null=True) + poverty = models.BooleanField(db_column='Poverty', blank=True, null=True) family_type = models.ForeignKey( FamilyType, models.DO_NOTHING, db_column='FamilyType', blank=True, null=True, related_name='families') address = models.CharField( db_column='FamilyAddress', max_length=200, blank=True, null=True) - is_offline = models.NullBooleanField( + is_offline = models.BooleanField( db_column='isOffline', blank=True, null=True) ethnicity = models.CharField( db_column='Ethnicity', max_length=1, blank=True, null=True) @@ -251,7 +248,6 @@ def is_adult(self, reference_date=None): health_facility = models.ForeignKey( location_models.HealthFacility, models.DO_NOTHING, db_column='HFID', blank=True, null=True, related_name='insurees') - json_ext = FallbackJSONField(db_column="JsonExt", blank=True, null=True) offline = models.BooleanField(db_column='isOffline', blank=True, null=True) audit_user_id = models.IntegerField(db_column='AuditUserID') diff --git a/insuree/schema.py b/insuree/schema.py index 67e81f4..e6784ab 100644 --- a/insuree/schema.py +++ b/insuree/schema.py @@ -98,7 +98,7 @@ class Query(graphene.ObjectType): ) def resolve_insuree_number_validity(self, info, **kwargs): - errors = validate_insuree_number(kwargs['insuree_number'], kwargs['new_insuree']) + errors = validate_insuree_number(kwargs['insuree_number'], kwargs.get('new_insuree', False)) if errors: return False else: From 11bb2c776fdf840713f514cb0ed9a0dd472b1de4 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Fri, 23 Sep 2022 11:32:14 +0200 Subject: [PATCH 14/14] Request the full path of a photo The config being module-specific and the path being based on two fields + config, a Model feature makes a lot of sense. --- insuree/models.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/insuree/models.py b/insuree/models.py index d292bed..090c7ce 100644 --- a/insuree/models.py +++ b/insuree/models.py @@ -1,3 +1,4 @@ +import os.path import uuid import core @@ -45,6 +46,11 @@ class InsureePhoto(core_models.VersionedModel): db_column='AuditUserID', blank=True, null=True) # rowid = models.TextField(db_column='RowID', blank=True, null=True) + def full_file_path(self): + if not InsureeConfig.insuree_photos_root_path or not self.filename: + return None + return os.path.join(InsureeConfig.insuree_photos_root_path, self.folder, self.filename) + class Meta: managed = False db_table = 'tblPhotos'