Merge openIMIS Main to Fork main

.github/workflows/openmis-module-test.yml

@@ -77,6 +77,7 @@ jobs:
 #          black --check .
 
       - name: Django tests
+        id: django_tests
         working-directory: ./openimis/openIMIS
         run: |
           export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')"
@@ -85,9 +86,11 @@ jobs:
           python manage.py migrate
           python init_test_db.py | grep . | uniq -c
           python manage.py test --keepdb $MODULE_NAME
+          if [ $? == 1 ]; then cat debug.log; fi
         env:
           SECRET_KEY: secret
           DEBUG: true
+          DJANGO_LOG_LEVEL: DEBUG
           #DJANGO_SETTINGS_MODULE: hat.settings
           DB_HOST: localhost
           DB_PORT: 1433
@@ -96,5 +99,8 @@ jobs:
           DB_PASSWORD: GitHub999
           #DEV_SERVER: true
           SITE_ROOT: api
-
-
+      - name: Print debug logs on failed tests
+        if: failure()
+        working-directory: ./openimis/openIMIS
+        run: |
+          cat debug.log

insuree/apps.py

@@ -1,3 +1,5 @@
+import os
+
 from django.apps import AppConfig
 from django.conf import settings
 
@@ -63,7 +65,6 @@ def _configure_permissions(self, cfg):
         InsureeConfig.gql_mutation_create_insurees_perms = cfg["gql_mutation_create_insurees_perms"]
         InsureeConfig.gql_mutation_update_insurees_perms = cfg["gql_mutation_update_insurees_perms"]
         InsureeConfig.gql_mutation_delete_insurees_perms = cfg["gql_mutation_delete_insurees_perms"]
-        InsureeConfig.insuree_photos_root_path = cfg["insuree_photos_root_path"]
         InsureeConfig.insuree_number_validator = cfg["insuree_number_validator"]
         InsureeConfig.insuree_number_length = cfg["insuree_number_length"]
         InsureeConfig.insuree_number_modulo_root = cfg["insuree_number_modulo_root"]
@@ -81,6 +82,7 @@ def ready(self):
         self._configure_permissions(cfg)
         self._configure_fake_insurees(cfg)
         self._configure_renewal(cfg)
+        self._configure_photo_root(cfg)
 
     # Getting these at runtime for easier testing
     @classmethod
@@ -106,3 +108,12 @@ def set_dataloaders(self, dataloaders):
     @classmethod
     def __get_from_settings_or_default(cls, attribute_name, default=None):
         return getattr(settings, attribute_name) if hasattr(settings, attribute_name) else default
+
+    def _configure_photo_root(self, cfg):
+        # TODO: To be confirmed. I left loading from config for integrity reasons
+        #  but it could be based on env variable only.
+        #  Also we could determine global file root for all stored files across modules.
+        if from_config := cfg.get("insuree_photos_root_path", None):
+            InsureeConfig.insuree_photos_root_path = from_config
+        elif from_env := os.getenv("PHOTO_ROOT_PATH", None):
+            InsureeConfig.insuree_photos_root_path = from_env

insuree/gql_mutations.py

@@ -33,7 +33,7 @@ class InsureeBase:
     chf_id = graphene.String(max_length=12, required=False)
     last_name = graphene.String(max_length=100, required=True)
     other_names = graphene.String(max_length=100, required=True)
-    gender_id = graphene.String(max_length=1, required=False)
+    gender_id = graphene.String(max_length=1, required=True, description="Was mandatory in Legacy but not in modular")
     dob = graphene.Date(required=True)
     head = graphene.Boolean(required=False)
     marital = graphene.String(max_length=1, required=False)
@@ -152,7 +152,7 @@ def async_mutate(cls, user, **data):
             data['validity_from'] = TimeUtils.now()
             client_mutation_id = data.get("client_mutation_id")
             # Validate insuree number right away
-            errors = validate_insuree_number(data.get("head_insuree", {}).get("chf_id", None))
+            errors = validate_insuree_number(data.get("head_insuree", {}).get("chf_id", None), True)
             if errors:
                 return errors
             family = update_or_create_family(data, user)
@@ -253,7 +253,7 @@ def async_mutate(cls, user, **data):
             data['validity_from'] = TimeUtils.now()
             client_mutation_id = data.get("client_mutation_id")
             # Validate insuree number right away
-            errors = validate_insuree_number(data.get("chf_id", None))
+            errors = validate_insuree_number(data.get("chf_id", None), True)
             if errors:
                 return errors
             insuree = update_or_create_insuree(data, user)

insuree/gql_queries.py

@@ -1,11 +1,15 @@
 import graphene
 from graphene_django import DjangoObjectType
+
+from .apps import InsureeConfig
 from .models import Insuree, InsureePhoto, Education, Profession, Gender, IdentificationType, \
     Family, FamilyType, ConfirmationType, Relation, InsureePolicy, FamilyMutation, InsureeMutation
 from location.schema import LocationGQLType
 from policy.gql_queries import PolicyGQLType
 from core import prefix_filterset, filter_validity, ExtendedConnection
 
+from .services import load_photo_file
+
 
 class GenderGQLType(DjangoObjectType):
     class Meta:
@@ -16,6 +20,15 @@ class Meta:
 
 
 class PhotoGQLType(DjangoObjectType):
+    photo = graphene.String()
+
+    def resolve_photo(self, info):
+        if self.photo:
+            return self.photo
+        elif InsureeConfig.insuree_photos_root_path and self.folder and self.filename:
+            return load_photo_file(self.folder, self.filename)
+        return None
+
     class Meta:
         model = InsureePhoto
         filter_fields = {
@@ -76,6 +89,7 @@ class Meta:
 class InsureeGQLType(DjangoObjectType):
     age = graphene.Int(source='age')
     client_mutation_id = graphene.String()
+    photo = PhotoGQLType()
 
     def resolve_current_village(self, info):
         if "location_loader" in info.context.dataloaders and self.current_village_id:
@@ -96,6 +110,9 @@ def resolve_health_facility(self, info):
             )
         return self.health_facility
 
+    def resolve_photo(self, info):
+        return self.photo
+
     class Meta:
         model = Insuree
         filter_fields = {

insuree/migrations/0007_auto_20200722_0940.py

@@ -1,4 +1,5 @@
 # Generated by Django 3.0.3 on 2020-07-22 09:40
+from django.conf import settings
 
 import core.fields
 import datetime
@@ -23,8 +24,12 @@ class Migration(migrations.Migration):
         # ),
         # so let's make it raw SQL...
         migrations.RunSQL(
-            "ALTER TABLE[tblInsuree] ADD CONSTRAINT[tblInsuree_CurrentVillage_8ea25085_fk_tblLocations_LocationId] \
-             FOREIGN KEY([CurrentVillage]) REFERENCES[tblLocations]([LocationId]);"
+            "ALTER TABLE [tblInsuree] ADD CONSTRAINT "
+            "[tblInsuree_CurrentVillage_8ea25085_fk_tblLocations_LocationId] "
+            "FOREIGN KEY([CurrentVillage]) REFERENCES[tblLocations]([LocationId]);"
+            if settings.MSSQL else
+            'ALTER TABLE "tblInsuree" ADD CONSTRAINT "tblInsuree_CurrentVillage_8ea25085_fk_tblLocations_LocationId" '
+            ' FOREIGN KEY("CurrentVillage") REFERENCES "tblLocations"("LocationId");'
         )
 
     ]

insuree/migrations/0008_auto_20200731_0443.py

@@ -1,5 +1,5 @@
 # Generated by Django 3.0.3 on 2020-07-31 04:43
-
+from django.conf import settings
 from django.db import migrations
 
 
@@ -12,5 +12,7 @@ class Migration(migrations.Migration):
     operations = [
         migrations.RunSQL(
             "ALTER TABLE[tblInsuree] ALTER COLUMN FamilyID [int] NULL;"
+            if settings.MSSQL else
+            'ALTER TABLE "tblInsuree" ALTER COLUMN "FamilyID" DROP NOT NULL;'
         )
     ]

insuree/migrations/0010_auto_20200731_0524.py

@@ -1,5 +1,5 @@
 # Generated by Django 3.0.3 on 2020-07-31 05:24
-
+from django.conf import settings
 from django.db import migrations
 
 
@@ -10,6 +10,10 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunSQL('ALTER TABLE tblInsuree ADD JsonExt TEXT'),
-        migrations.RunSQL('ALTER TABLE tblFamilies ADD JsonExt TEXT'),
+        migrations.RunSQL('ALTER TABLE [tblInsuree] ADD [JsonExt] TEXT'
+                          if settings.MSSQL else
+                          'ALTER TABLE "tblInsuree" ADD "JsonExt" jsonb'),
+        migrations.RunSQL('ALTER TABLE [tblFamilies] ADD [JsonExt] TEXT'
+                          if settings.MSSQL else
+                          'ALTER TABLE "tblFamilies" ADD "JsonExt" jsonb'),
     ]

insuree/migrations/0011_auto_20200807_1309.py

@@ -1,4 +1,5 @@
 # Generated by Django 3.0.3 on 2020-08-07 13:09
+from django.conf import settings
 
 import core.fields
 from django.db import migrations, models
@@ -12,9 +13,23 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunSQL('ALTER TABLE tblPhotos ADD LegacyID [int] NULL, photo [TEXT] null'),
-        migrations.RunSQL('ALTER TABLE tblPhotos ALTER COLUMN PhotoFolder nvarchar(255) NULL'),
-        migrations.RunSQL('ALTER TABLE tblPhotos ALTER COLUMN PhotoFileName nvarchar(255) NULL'),
+        migrations.RunSQL(
+            'ALTER TABLE "tblPhotos" ADD "LegacyID" int NULL, photo TEXT null'
+            if settings.MSSQL else
+            'ALTER TABLE "tblPhotos" ADD "LegacyID" int; ALTER TABLE "tblPhotos" add photo TEXT;'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFolder" nvarchar(255) NULL'
+            if settings.MSSQL else
+            'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFolder" TYPE VARCHAR(255); '
+            'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFolder" DROP NOT NULL;'
+        ),
+        migrations.RunSQL(
+            'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFileName" nvarchar(255) NULL'
+            if settings.MSSQL else
+            'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFileName" TYPE VARCHAR(255);'
+            'ALTER TABLE "tblPhotos" ALTER COLUMN "PhotoFileName" DROP NOT NULL;'
+        ),
         migrations.CreateModel(
             name='InsureePhoto',
             fields=[

insuree/migrations/0013_auto_20211103_1023.py

@@ -1,5 +1,5 @@
 # Generated by Django 3.0.14 on 2021-11-03 10:23
-
+from django.conf import settings
 from django.db import migrations
 
 
@@ -12,12 +12,18 @@ class Migration(migrations.Migration):
     operations = [
         migrations.RunSQL(
             """
-        CREATE NONCLUSTERED INDEX ix_tblInsuree_validity ON [dbo].[tblInsuree]
-        (
-            [ValidityFrom] ASC,
-            [LegacyID] ASC,
-            InsureeId
-        )""",
-            reverse_sql="DROP index ix_tblInsuree_validity on tblInsuree",
+            CREATE NONCLUSTERED INDEX ix_tblInsuree_validity ON [dbo].[tblInsuree]
+            (
+                [ValidityFrom] ASC,
+                [LegacyID] ASC,
+                InsureeID
+            )""" if settings.MSSQL else """
+            CREATE INDEX "ix_tblInsuree_validity" ON "tblInsuree"
+            (
+                "ValidityFrom" ASC,
+                "LegacyID" ASC,
+                "InsureeID"
+            )""",
+            reverse_sql='DROP index "ix_tblInsuree_validity" on "tblInsuree"',
         )
     ]

insuree/models.py

@@ -1,3 +1,4 @@
+import os.path
 import uuid
 
 import core
@@ -45,6 +46,11 @@ class InsureePhoto(core_models.VersionedModel):
         db_column='AuditUserID', blank=True, null=True)
     # rowid = models.TextField(db_column='RowID', blank=True, null=True)
 
+    def full_file_path(self):
+        if not InsureeConfig.insuree_photos_root_path or not self.filename:
+            return None
+        return os.path.join(InsureeConfig.insuree_photos_root_path, self.folder, self.filename)
+
     class Meta:
         managed = False
         db_table = 'tblPhotos'
@@ -89,14 +95,13 @@ class Family(core_models.VersionedModel, core_models.ExtendableModel):
     location = models.ForeignKey(
         location_models.Location,
         models.DO_NOTHING, db_column='LocationId', blank=True, null=True)
-    # Need to be NullBooleanField (BooleanField + null=True is not enough) for Graphene to map properly
-    poverty = models.NullBooleanField(db_column='Poverty', blank=True, null=True)
+    poverty = models.BooleanField(db_column='Poverty', blank=True, null=True)
     family_type = models.ForeignKey(
         FamilyType, models.DO_NOTHING, db_column='FamilyType', blank=True, null=True,
         related_name='families')
     address = models.CharField(
         db_column='FamilyAddress', max_length=200, blank=True, null=True)
-    is_offline = models.NullBooleanField(
+    is_offline = models.BooleanField(
         db_column='isOffline', blank=True, null=True)
     ethnicity = models.CharField(
         db_column='Ethnicity', max_length=1, blank=True, null=True)
@@ -297,7 +302,7 @@ class Meta:
 class InsureePolicy(core_models.VersionedModel):
     id = models.AutoField(db_column='InsureePolicyID', primary_key=True)
 
-    insuree = models.ForeignKey(Insuree, models.DO_NOTHING, db_column='InsureeId', related_name="insuree_policies")
+    insuree = models.ForeignKey(Insuree, models.DO_NOTHING, db_column='InsureeID', related_name="insuree_policies")
     policy = models.ForeignKey("policy.Policy", models.DO_NOTHING, db_column='PolicyId',
                                related_name="insuree_policies")
 

insuree/reports/insuree_family_overview.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from django.db.models import Q, F
 
 # If manually pasting from reportbro and you have test data, search and replace \" with \\"
@@ -1709,16 +1710,17 @@ def insuree_family_overview_query(user, date_from=None, date_to=None, **kwargs):
     if date_to:
         filters &= Q(validity_from__lte=date_to + datetimedelta(days=1))
 
-    # TODO use auth from Quentin's PR
-    # if settings.ROW_SECURITY:
-    #     from location.models import UserDistrict
-    #     dist = UserDistrict.get_user_districts(user._u)
-    #     queryset = queryset.filter(
-    #         health_facility__location__id__in=[l.location_id for l in dist]
-    #     )
+    if settings.ROW_SECURITY:
+        from location.models import UserDistrict
+        dist = UserDistrict.get_user_districts(user._u)
+        queryset = Insuree.objects.filter(
+            health_facility__location__id__in=[l.location_id for l in dist]
+        )
+    else:
+        queryset = Insuree.objects
 
     queryset = (
-        Insuree.objects.filter(filters)
+        queryset.filter(filters)
         .values(
             "chf_id",
             "other_names",

insuree/schema.py

@@ -93,11 +93,12 @@ class Query(graphene.ObjectType):
     insuree_number_validity = graphene.Field(
         graphene.Boolean,
         insuree_number=graphene.String(required=True),
+        new_insuree=graphene.Boolean(required=False),
         description="Checks that the specified insuree number is valid"
     )
 
-    def resolve_insuree_number_validity(self, info, insuree_number=None):
-        errors = validate_insuree_number(insuree_number)
+    def resolve_insuree_number_validity(self, info, **kwargs):
+        errors = validate_insuree_number(kwargs['insuree_number'], kwargs.get('new_insuree', False))
         if errors:
             return False
         else:
@@ -151,7 +152,6 @@ def resolve_insurees(self, info, **kwargs):
                         (Q(current_village__isnull=True) & Q(**{family_location: parent_location}))]
         return gql_optimizer.query(Insuree.objects.filter(*filters).all(), info)
 
-
     def resolve_family_members(self, info, **kwargs):
         if not info.context.user.has_perms(InsureeConfig.gql_query_insurees_perms):
             raise PermissionDenied(_("unauthorized"))