#2714 day zero fields

data-serving/reusable-data-service/data_service/controller/case_controller.py

@@ -163,6 +163,9 @@ def generate_output():
 
         return generate_output
 
+    def valid_statuses(self):
+        return [f.values for f in Case.custom_fields if f.key == "caseStatus"][0]
+
     def batch_status_change(
         self,
         status: str,
@@ -172,18 +175,18 @@ def batch_status_change(
     ):
         """Update all of the cases identified in case_ids to have the supplied curation status.
         Raises PreconditionUnsatisfiedError or ValidationError on invalid input."""
-        statuses = CaseReference.valid_statuses()
+        statuses = self.valid_statuses()
         if not status in statuses:
             raise PreconditionUnsatisfiedError(f"status {status} not one of {statuses}")
         if filter is not None and case_ids is not None:
             raise PreconditionUnsatisfiedError(
                 "Do not supply both a filter and a list of IDs"
             )
-        if status == "EXCLUDED" and note is None:
+        if status == "omit_error" and note is None:
             raise ValidationError(f"Excluding cases must be documented in a note")
 
         def update_status(id: str, status: str, note: str):
-            if status == "EXCLUDED":
+            if status == "omit_error":
                 caseExclusion = CaseExclusionMetadata()
                 caseExclusion.note = note
             else:

data-serving/reusable-data-service/data_service/controller/schema_controller.py

@@ -1,7 +1,7 @@
 import dataclasses
 
 from datetime import date
-from typing import Optional, Union
+from typing import Any, List, Optional, Union
 
 from data_service.model.case import add_field_to_case_class, observe_case_class
 from data_service.model.field import Field
@@ -33,6 +33,7 @@ def restore_saved_fields(self) -> None:
                 field.data_dictionary_text,
                 field.required,
                 field.default,
+                field.values,
                 False,
             )
 
@@ -43,6 +44,7 @@ def add_field(
         description: str,
         required: bool = False,
         default: Optional[Union[bool, str, int, date]] = None,
+        values: Optional[List[Any]] = None,
         store_field: bool = True,
     ):
         global Case
@@ -61,7 +63,7 @@ def add_field(
         If a field is required, set required = True. You must also set a default value so that
         existing cases have an initial setting for the field."""
         required = required if required is not None else False
-        field_model = Field(name, type_name, description, required, default)
+        field_model = Field(name, type_name, description, required, default, values)
         add_field_to_case_class(field_model)
         if store_field:
             self.store.add_field(field_model)

data-serving/reusable-data-service/data_service/day_zero_fields.json

@@ -5,6 +5,30 @@
         "data_dictionary_text": "A unique identifier for this case, in the form of a mongoDB object identifier (24 characters 0-9a-f).",
         "required": false
     },
+    {
+        "key": "caseStatus",
+        "type": "string",
+        "data_dictionary_text": "Status of a case. Cases which are discarded were previously suspected but have now been confirmed negative, and should be excluded from case counts. Cases which are omit_error were incorrectly added and should be dismissed from any data interpretation.",
+        "required": true,
+        "values": [
+            "confirmed",
+            "probable",
+            "suspected",
+            "discarded",
+            "omit_error"
+        ]
+    },
+    {
+        "key": "pathogenStatus",
+        "type": "string",
+        "data_dictionary_text": "Whether the infection occured in an endemic, or non-endemic region.",
+        "required": "true",
+        "values": [
+            "endemic",
+            "emerging",
+            "unknown"
+        ]
+    },
     {
         "key": "confirmationDate",
         "type": "date",

data-serving/reusable-data-service/data_service/main.py

@@ -163,6 +163,7 @@ def add_field_to_case_schema():
             req["description"],
             req.get("required"),
             req.get("default"),
+            req.get("values"),
         )
         return "", 201
     except WebApplicationError as e:

data-serving/reusable-data-service/data_service/model/case_reference.py

@@ -10,7 +10,6 @@ class CaseReference(Document):
 
     _: dataclasses.KW_ONLY
     sourceId: str = dataclasses.field(init=False, default=None)
-    status: str = dataclasses.field(init=False, default="UNVERIFIED")
 
     def validate(self):
         """Check whether I am consistent. Raise ValueError if not."""
@@ -19,18 +18,10 @@ def validate(self):
             raise ValueError("Source ID is mandatory")
         elif self.sourceId is None:
             raise ValueError("Source ID must have a value")
-        if self.status not in self.valid_statuses():
-            raise ValueError(f"Status {self.status} is not acceptable")
-
-    @staticmethod
-    def valid_statuses():
-        """A case reference must have one of these statuses."""
-        return ["EXCLUDED", "UNVERIFIED", "VERIFIED"]
 
     @staticmethod
     def from_dict(d: dict[str, str]):
         """Create a CaseReference from a dictionary representation."""
         ref = CaseReference()
         ref.sourceId = d.get("sourceId")
-        ref.status = d.get("status", "UNVERIFIED")
         return ref

data-serving/reusable-data-service/data_service/model/document.py

@@ -193,10 +193,16 @@ def validate(self):
         """Check whether I am consistent. Raise ValidationError if not."""
         for field in self.custom_fields:
             getter = operator.attrgetter(field.key)
-            if field.required is True and getter(self) is None:
+            value = getter(self)
+            if field.required is True and value is None:
                 raise ValidationError(f"{field.key} must have a value")
-            if field.key in self.document_fields() and getter(self) is not None:
+            if field.key in self.document_fields() and value is not None:
                 getter(self).validate()
+            if field.values is not None:
+                if value is not None and value not in field.values:
+                    raise ValidationError(
+                        f"{field.key} value {value} not in permissible values {field.values}"
+                    )
 
     def _internal_set_value(self, key, value):
         self._internal_ensure_containers_exist(key)

data-serving/reusable-data-service/data_service/model/field.py

@@ -1,6 +1,6 @@
 import dataclasses
 from datetime import date
-from typing import Optional, Union
+from typing import Any, List, Optional, Union
 
 from data_service.model.case_exclusion_metadata import CaseExclusionMetadata
 from data_service.model.case_reference import CaseReference
@@ -20,6 +20,8 @@ class Field(Document):
     default: Optional[Union[bool, str, int, date]] = dataclasses.field(
         init=True, default=None
     )
+    values: Optional[List[Any]] = dataclasses.field(init=True, default=None)
+
     STRING = "string"
     DATE = "date"
     INTEGER = "integer"
@@ -49,6 +51,7 @@ def from_dict(cls, dictionary):
             dictionary.get("data_dictionary_text"),
             dictionary.get("required"),
             dictionary.get("default", None),
+            dictionary.get("values", None),
         )
 
     def python_type(self) -> type:

data-serving/reusable-data-service/data_service/stores/memory_store.py

@@ -53,7 +53,7 @@ def update_case_status(
         self, id: str, status: str, exclusion: CaseExclusionMetadata
     ):
         case = self.case_by_id(id)
-        case.caseReference.status = status
+        case.caseStatus = status
         case.caseExclusion = exclusion
 
     def fetch_cases(self, page: int, limit: int, predicate: Filter):
@@ -71,8 +71,7 @@ def excluded_cases(self, source_id: str, filter: Filter):
         return [
             c
             for c in self.cases.values()
-            if c.caseReference.sourceId == source_id
-            and c.caseReference.status == "EXCLUDED"
+            if c.caseReference.sourceId == source_id and c.caseStatus == "omit_error"
         ]
 
     def delete_case(self, case_id: str):

data-serving/reusable-data-service/data_service/stores/mongo_store.py

@@ -91,7 +91,7 @@ def replace_case(self, id: str, case: Case):
     def update_case_status(
         self, id: str, status: str, exclusion: CaseExclusionMetadata
     ):
-        update = {"$set": {"caseReference.status": status}}
+        update = {"$set": {"caseStatus": status}}
         if exclusion:
             update["$set"][
                 "caseExclusion"
@@ -124,7 +124,7 @@ def excluded_cases(self, source_id: str, filter: Filter) -> List[Case]:
                 "$and": [
                     {
                         "caseReference.sourceId": ObjectId(source_id),
-                        "caseReference.status": "EXCLUDED",
+                        "caseStatus": "omit_error",
                     },
                     query,
                 ]

data-serving/reusable-data-service/tests/data/case.excluded.json

@@ -0,0 +1,12 @@
+{
+    "confirmationDate": "2022-05-01T01:23:45.678Z",
+    "caseReference": {
+        "sourceId": "fedc12345678901234567890"
+    },
+    "caseExclusion": {
+        "date": "2022-06-01T01:23:45.678Z",
+        "note": "Excluded upon this day, for reasons"
+    },
+    "caseStatus": "omit_error",
+    "pathogenStatus": "endemic"
+}

data-serving/reusable-data-service/tests/data/case.minimal.json

@@ -2,5 +2,7 @@
     "confirmationDate": "2021-12-31T01:23:45.678Z",
     "caseReference": {
         "sourceId": "fedc09876543210987654321"
-    }
+    },
+    "caseStatus": "probable",
+    "pathogenStatus": "emerging"
 }

data-serving/reusable-data-service/tests/data/case.with_location.json

@@ -15,5 +15,7 @@
         "properties": {
             "country": "IND"
         }
-    }
+    },
+    "caseStatus": "probable",
+    "pathogenStatus": "unknown"
 }