#2714 age ranges

data-serving/reusable-data-service/data_service/day_zero_fields.json

@@ -29,6 +29,18 @@
             "unknown"
         ]
     },
+    {
+        "key": "location",
+        "type": "geofeature",
+        "data_dictionary_text": "The location associated with this case.",
+        "required": false
+    },
+    {
+        "key": "age",
+        "type": "age_range",
+        "data_dictionary_text": "Age of the individual, specified as a range, either open-ended (<n, >n) or as a range delimited by a hyphen following 5-year age increments (m-n)",
+        "required": false
+    },
     {
         "key": "confirmationDate",
         "type": "date",
@@ -46,11 +58,5 @@
         "type": "CaseExclusion",
         "data_dictionary_text": "If this case is excluded from the line list, information about when and why it was excluded.",
         "required": false
-    },
-    {
-        "key": "location",
-        "type": "geofeature",
-        "data_dictionary_text": "The location associated with this case.",
-        "required": false
     }
 ]

data-serving/reusable-data-service/data_service/model/age_range.py

@@ -0,0 +1,56 @@
+import dataclasses
+
+from data_service.model.document import Document
+from data_service.util.errors import ValidationError
+
+@dataclasses.dataclass
+class AgeRange(Document):
+    """I represent a numerical range within which a person's age lies (inclusive of both limits).
+    To avoid reidentifying people who have been anonymised by this
+    application, I will only tell you their age to within five years (unless they are infants)."""
+    lower: int = None
+    upper: int = None
+
+    def __post_init__(self):
+        """Massage the supplied lower and upper bounds to fit our requirements. That doesn't
+        preclude somebody changing the values after initialisation so we also fix in validate()."""
+        self.fix_my_boundaries()
+
+    def fix_my_boundaries(self):
+        if self.lower is not None and self.lower != 0:
+            self.lower = (self.lower // 5) * 5 + 1
+        if self.upper is not None and self.upper != 1 and self.upper % 5 != 0:
+            self.upper = ((self.upper // 5) + 1) * 5
+
+    def validate(self):
+        """I must represent the range [0,1], or a range greater than five years, and must
+        have a positive lower bound and an upper bound below 121."""
+        self.fix_my_boundaries()
+        if self.lower is None:
+            raise ValidationError("Age Range must have a lower bound")
+        if self.upper is None:
+            raise ValidationError("Age Range must have an upper bound")
+        if self.lower < 0:
+            raise ValidationError(f"Lower bound {self.lower} is below the minimum permissible 0")
+        if self.upper < 1:
+            raise ValidationError(f"Upper bound {self.upper} is below the minimum permissible 1")
+        if self.upper > 120:
+            raise ValidationError(f"Upper bound {self.upper} is above the maximum permissible 120")
+        # deal with the special case first
+        if self.lower == 0 and self.upper == 1:
+            return
+        if self.upper - self.lower < 4:
+            # remember range is inclusive of bounds so e.g. 1-5 is five years
+            raise ValidationError(f"Range [{self.lower}, {self.upper}] is too small")
+
+    @classmethod
+    def from_dict(cls, dict_description):
+        ages = cls()
+        # age ranges can be open-ended according to the data dictionary, which we map onto our absolute limits
+        ages.lower = dict_description.get('lower', 0)
+        ages.upper = dict_description.get('upper', 120)
+        return ages
+
+    @classmethod
+    def none_field_values(cls):
+        return ['', '']

data-serving/reusable-data-service/data_service/model/document.py

@@ -118,7 +118,10 @@ def field_values(self) -> List[str]:
             value = getattr(self, f.name)
             if issubclass(f.type, Document):
                 if self.include_dataclass_fields(f.type):
-                    fields += value.field_values()
+                    if value is not None:
+                        fields += value.field_values()
+                    else:
+                        fields += f.type.none_field_values()
             elif hasattr(f.type, "custom_field_names"):
                 if value is not None:
                     fields += value.custom_field_values()

data-serving/reusable-data-service/data_service/model/field.py

@@ -2,6 +2,7 @@
 from datetime import date
 from typing import Any, List, Optional, Union
 
+from data_service.model.age_range import AgeRange
 from data_service.model.case_exclusion_metadata import CaseExclusionMetadata
 from data_service.model.case_reference import CaseReference
 from data_service.model.document import Document
@@ -26,11 +27,13 @@ class Field(Document):
     DATE = "date"
     INTEGER = "integer"
     LOCATION = "geofeature"
+    AGE_RANGE = "age_range"
     type_map = {
         STRING: str,
         DATE: date,
         INTEGER: int,
         LOCATION: Feature,
+        AGE_RANGE: AgeRange,
         "CaseReference": CaseReference,
         "CaseExclusion": CaseExclusionMetadata,
     }

data-serving/reusable-data-service/tests/test_age_range.py

@@ -0,0 +1,60 @@
+import pytest
+
+from data_service.model.age_range import AgeRange
+from data_service.util.errors import ValidationError
+
+from tests.util import does_not_raise
+
+
+def test_age_range_massages_input_to_match_buckets():
+    ages = AgeRange(2,6)
+    assert ages.lower == 1
+    assert ages.upper == 10
+
+
+def test_age_range_leaves_input_if_it_is_already_on_bucket_boundary():
+    ages = AgeRange(6, 10)
+    assert ages.lower == 6
+    assert ages.upper == 10
+
+
+def test_age_range_invalid_if_boundaries_are_None():
+    ages = AgeRange(None, None)
+    with pytest.raises(ValidationError):
+        ages.validate()
+    ages = AgeRange(None, 12)
+    with pytest.raises(ValidationError):
+        ages.validate()
+    ages = AgeRange(5, None)
+    with pytest.raises(ValidationError):
+        ages.validate()
+
+
+def test_age_range_invalid_if_lower_bound_negative():
+    ages = AgeRange(-12, 4)
+    with pytest.raises(ValidationError):
+        ages.validate()
+
+
+def test_age_range_invalid_if_upper_bound_negative():
+    ages = AgeRange(5, -27)
+    with pytest.raises(ValidationError):
+        ages.validate()
+
+
+def test_age_range_invalid_if_upper_bound_methuselan():
+    ages = AgeRange(15, 150)
+    with pytest.raises(ValidationError):
+        ages.validate()
+
+
+def test_age_range_ok_for_infants():
+    ages = AgeRange(0, 1)
+    with does_not_raise(ValidationError):
+        ages.validate()
+
+
+def test_age_range_ok_for_large_positive_range():
+    ages = AgeRange(30, 45)
+    with does_not_raise(ValidationError):
+        ages.validate()

data-serving/reusable-data-service/tests/test_case_end_to_end.py

@@ -171,6 +171,25 @@ def test_post_case_list_cases_geojson_round_trip(client_with_patched_mongo):
     assert get_response.json["cases"][0]["location"]["properties"]["country"] == "IND"
 
 
+def test_post_case_list_cases_with_age_round_trip(client_with_patched_mongo):
+    with open("./tests/data/case.minimal.json") as case_file:
+        case_doc = json.load(case_file)
+    case_doc['age'] = {
+        'lower': 4,
+        'upper': 12,
+    }
+    post_response = client_with_patched_mongo.post(
+        "/api/cases",
+        json=case_doc,
+    )
+    assert post_response.status_code == 201
+    get_response = client_with_patched_mongo.get("/api/cases")
+    assert get_response.status_code == 200
+    assert len(get_response.json["cases"]) == 1
+    assert get_response.json["cases"][0]["age"]["lower"] == 1
+    assert get_response.json["cases"][0]["age"]["upper"] == 15
+
+
 def test_post_multiple_case_list_cases_round_trip(client_with_patched_mongo):
     with open("./tests/data/case.minimal.json") as case_file:
         case_doc = json.load(case_file)

data-serving/reusable-data-service/tests/test_case_model.py

@@ -29,10 +29,9 @@ def test_case_with_geojson_is_valid():
 
 def test_csv_header():
     header_line = Case.csv_header()
-    assert (
-        header_line
-        == "_id,caseStatus,pathogenStatus,confirmationDate,caseReference.sourceId,location.country,location.latitude,location.longitude,location.admin1,location.admin2,location.admin3\r\n"
-    )
+    header_fields = header_line.split(',')
+    assert 'caseStatus' in header_fields
+    assert 'location.latitude' in header_fields
 
 
 def test_csv_row_with_no_id():
@@ -46,7 +45,9 @@ def test_csv_row_with_no_id():
     case.caseStatus = "probable"
     case.pathogenStatus = "emerging"
     csv = case.to_csv()
-    assert csv == ",probable,emerging,2022-06-13,abcd12903478565647382910,,,,,,\r\n"
+    csv_fields = csv.split(',')
+    assert 'probable' in csv_fields
+    assert '2022-06-13' in csv_fields
 
 
 def test_csv_row_with_id():
@@ -62,7 +63,10 @@ def test_csv_row_with_id():
     case.caseStatus = "probable"
     case.pathogenStatus = "unknown"
     csv = case.to_csv()
-    assert csv == f"{id1},probable,unknown,2022-06-13,{id2},,,,,,\r\n"
+    csv = case.to_csv()
+    csv_fields = csv.split(',')
+    assert 'probable' in csv_fields
+    assert '2022-06-13' in csv_fields
 
 
 def test_apply_update_to_case():

data-serving/reusable-data-service/tests/test_geojson_model.py

@@ -1,18 +1,9 @@
 import pytest
 
-from contextlib import contextmanager
-
 from data_service.model.geojson import Point, Feature
 from data_service.util.errors import ValidationError
 
-
-@contextmanager
-def does_not_raise(exception):
-    try:
-        yield
-    except exception:
-        raise pytest.fail(f"Exception {exception} expected not to be raised")
-
+from tests.util import does_not_raise
 
 def test_point_needs_two_coordinates():
     p = Point()

data-serving/reusable-data-service/tests/util.py

@@ -0,0 +1,10 @@
+import pytest
+
+from contextlib import contextmanager
+
+@contextmanager
+def does_not_raise(exception):
+    try:
+        yield
+    except exception:
+        raise pytest.fail(f"Exception {exception} expected not to be raised")