Merge pull request #410 from Crunch-io/add-category-date

Add category date

.github/workflows/test-and-deploy.yaml

@@ -20,7 +20,6 @@ jobs:
       matrix:
         python-version: 
           - 2.7
-          - 3.5
           - 3.6
           - 3.7
     steps:

scrunch/categories.py

@@ -9,6 +9,7 @@ class Category(ReadOnly):
     _MUTABLE_ATTRIBUTES = {'name', 'numeric_value', 'missing', 'selected', 'date'}
     _IMMUTABLE_ATTRIBUTES = {'id'}
     _ENTITY_ATTRIBUTES = _MUTABLE_ATTRIBUTES | _IMMUTABLE_ATTRIBUTES
+    _NULLABLE_ATTRIBUTES = {"date", "numeric_value", "selected"}
 
     def __init__(self, variable_resource, category):
         super(Category, self).__init__(variable_resource)
@@ -19,7 +20,13 @@ def __getattr__(self, item):
             if item == 'selected':
                 # Default is False; not always present
                 return self._category.get('selected', False)
-            return self._category[item]  # Has to exist
+            try:
+                return self._category[item]
+            except KeyError:
+                if item in self._NULLABLE_ATTRIBUTES:
+                    return None
+                else:
+                    raise  # API returned an incomplete category? Error
 
         # Attribute doesn't exist, must raise an AttributeError
         raise AttributeError(

scrunch/datasets.py

@@ -35,7 +35,7 @@
                              case_expr, download_file, shoji_entity_wrapper,
                              subvar_alias, validate_categories, shoji_catalog_wrapper,
                              get_else_case, else_case_not_selected, SELECTED_ID,
-                             NOT_SELECTED_ID, NO_DATA_ID)
+                             NOT_SELECTED_ID, NO_DATA_ID, valid_categorical_date)
 from scrunch.order import DatasetVariablesOrder, ProjectDatasetsOrder
 from scrunch.subentity import Deck, Filter, Multitable
 from scrunch.variables import (combinations_from_map, combine_categories_expr,
@@ -3055,7 +3055,7 @@ def integrate(self):
             self.resource.edit(derived=False)
             self.dataset._reload_variables()
 
-    def add_category(self, id, name, numeric_value, missing=False, before_id=False):
+    def add_category(self, id, name, numeric_value, missing=False, date=None, before_id=False):
         if self.resource.body['type'] not in CATEGORICAL_TYPES:
             raise TypeError(
                 "Variable of type %s do not have categories"
@@ -3065,6 +3065,18 @@ def add_category(self, id, name, numeric_value, missing=False, before_id=False):
             raise TypeError("Cannot add categories on derived variables. Re-derive with the appropriate expression")
 
         categories = self.resource.body['categories']
+        category_data = {
+            'id': id,
+            'missing': missing,
+            'name': name,
+            'numeric_value': numeric_value,
+        }
+        if date is not None:
+            if not isinstance(date, six.string_types):
+                raise ValueError("Date must be a string")
+            if not valid_categorical_date(date):
+                raise ValueError("Date must conform to Y-m-d format")
+            category_data["date"] = date
 
         if before_id:
             # only accept int type
@@ -3079,21 +3091,11 @@ def add_category(self, id, name, numeric_value, missing=False, before_id=False):
             new_categories = []
             for category in categories:
                 if category['id'] == before_id:
-                    new_categories.append({
-                        'id': id,
-                        'missing': missing,
-                        'name': name,
-                        'numeric_value': numeric_value,
-                    })
+                    new_categories.append(category_data)
                 new_categories.append(category)
             categories = new_categories
         else:
-            categories.append({
-                'id': id,
-                'missing': missing,
-                'name': name,
-                'numeric_value': numeric_value,
-            })
+            categories.append(category_data)
 
         resp = self.resource.edit(categories=categories)
         self._reload_variables()

scrunch/helpers.py

@@ -1,5 +1,6 @@
 import requests
 import six
+from datetime import datetime
 
 if six.PY2:  # pragma: no cover
     from urlparse import urljoin
@@ -232,3 +233,25 @@ def shoji_catalog_wrapper(index, **kwargs):
     }
     payload.update(**kwargs)
     return payload
+
+
+def valid_categorical_date(date_str):
+    """
+    Categories accept a `date` attribute that needs to be a valid ISO8601 date.
+    In order to keep dependencies reduced (no dateutil) and Python2x support,
+    we will support a limited set of simple date formats.
+    """
+    valid_date_masks = [
+        "%Y",
+        "%Y-%m",
+        "%Y-%m-%d",
+    ]
+    for mask in valid_date_masks:
+        try:
+            datetime.strptime(date_str, mask)
+            return True
+        except ValueError:
+            # Did not validate for this mask
+            continue
+    return False
+

scrunch/tests/test_datasets.py

@@ -1437,6 +1437,36 @@ def test_add_category(self):
         var.add_category(2, 'New category', 2, before_id=9)
         var.resource._edit.assert_called_with(categories=var.resource.body['categories'])
 
+    def test_add_category_date(self):
+        ds_mock = self._dataset_mock()
+        ds = BaseDataset(ds_mock)
+        var = ds['var4_alias']
+        var.resource.body['type'] = 'categorical'
+        categories = [
+            {"id": 1, "name": "Female", "missing": False, "numeric_value": 1},
+            {"id": 8, "name": "Male", "missing": False, "numeric_value": 8},
+            {"id": 9, "name": "No Data", "missing": True, "numeric_value": 9}
+        ]
+        var.resource.body['categories'] = categories[:]
+
+        with pytest.raises(ValueError) as err:
+            var.add_category(2, 'New category', 2, date=object())
+        assert str(err.value) == "Date must be a string"
+
+        with pytest.raises(ValueError) as err:
+            var.add_category(2, 'New category', 2, date="invalid date")
+        assert str(err.value) == "Date must conform to Y-m-d format"
+
+        var.add_category(2, 'New category', 2, date="2021-12-12")
+        new_categories = categories[:] + [{
+            "id": 2,
+            "name": "New category",
+            "date": "2021-12-12",
+            "numeric_value": 2,
+            "missing": False
+        }]
+        var.resource._edit.assert_called_with(categories=new_categories)
+
     def test_integrate_variables(self):
         ds_mock = mock.MagicMock()
         var_tuple = mock.MagicMock()

tox.ini

@@ -1,13 +1,12 @@
 [tox]
-envlist = py27,py34,py35,py36,py37,{py27,py36,py37}-pandas
+envlist = py27,py34,py36,py37,{py27,py36,py37}-pandas
 minversion = 2.4
 skip_missing_interpreters = true
 
 [gh-actions]
 python =
     2.7: py27,py27-pandas
     3.4: py34
-    3.5: py35,py35-pandas
     3.6: py36,py36-pandas
     3.7: py37,py37-pandas