diff --git a/.github/workflows/test-and-deploy.yaml b/.github/workflows/test-and-deploy.yaml index f949d118..5bdd7cf8 100644 --- a/.github/workflows/test-and-deploy.yaml +++ b/.github/workflows/test-and-deploy.yaml @@ -20,7 +20,6 @@ jobs: matrix: python-version: - 2.7 - - 3.5 - 3.6 - 3.7 steps: diff --git a/scrunch/categories.py b/scrunch/categories.py index 192962ed..899a4f79 100644 --- a/scrunch/categories.py +++ b/scrunch/categories.py @@ -9,6 +9,7 @@ class Category(ReadOnly): _MUTABLE_ATTRIBUTES = {'name', 'numeric_value', 'missing', 'selected', 'date'} _IMMUTABLE_ATTRIBUTES = {'id'} _ENTITY_ATTRIBUTES = _MUTABLE_ATTRIBUTES | _IMMUTABLE_ATTRIBUTES + _NULLABLE_ATTRIBUTES = {"date", "numeric_value", "selected"} def __init__(self, variable_resource, category): super(Category, self).__init__(variable_resource) @@ -19,7 +20,13 @@ def __getattr__(self, item): if item == 'selected': # Default is False; not always present return self._category.get('selected', False) - return self._category[item] # Has to exist + try: + return self._category[item] + except KeyError: + if item in self._NULLABLE_ATTRIBUTES: + return None + else: + raise # API returned an incomplete category? Error # Attribute doesn't exist, must raise an AttributeError raise AttributeError( diff --git a/scrunch/datasets.py b/scrunch/datasets.py index 6ea395c8..b326853e 100644 --- a/scrunch/datasets.py +++ b/scrunch/datasets.py @@ -35,7 +35,7 @@ case_expr, download_file, shoji_entity_wrapper, subvar_alias, validate_categories, shoji_catalog_wrapper, get_else_case, else_case_not_selected, SELECTED_ID, - NOT_SELECTED_ID, NO_DATA_ID) + NOT_SELECTED_ID, NO_DATA_ID, valid_categorical_date) from scrunch.order import DatasetVariablesOrder, ProjectDatasetsOrder from scrunch.subentity import Deck, Filter, Multitable from scrunch.variables import (combinations_from_map, combine_categories_expr, @@ -3055,7 +3055,7 @@ def integrate(self): self.resource.edit(derived=False) self.dataset._reload_variables() - def add_category(self, id, name, numeric_value, missing=False, before_id=False): + def add_category(self, id, name, numeric_value, missing=False, date=None, before_id=False): if self.resource.body['type'] not in CATEGORICAL_TYPES: raise TypeError( "Variable of type %s do not have categories" @@ -3065,6 +3065,18 @@ def add_category(self, id, name, numeric_value, missing=False, before_id=False): raise TypeError("Cannot add categories on derived variables. Re-derive with the appropriate expression") categories = self.resource.body['categories'] + category_data = { + 'id': id, + 'missing': missing, + 'name': name, + 'numeric_value': numeric_value, + } + if date is not None: + if not isinstance(date, six.string_types): + raise ValueError("Date must be a string") + if not valid_categorical_date(date): + raise ValueError("Date must conform to Y-m-d format") + category_data["date"] = date if before_id: # only accept int type @@ -3079,21 +3091,11 @@ def add_category(self, id, name, numeric_value, missing=False, before_id=False): new_categories = [] for category in categories: if category['id'] == before_id: - new_categories.append({ - 'id': id, - 'missing': missing, - 'name': name, - 'numeric_value': numeric_value, - }) + new_categories.append(category_data) new_categories.append(category) categories = new_categories else: - categories.append({ - 'id': id, - 'missing': missing, - 'name': name, - 'numeric_value': numeric_value, - }) + categories.append(category_data) resp = self.resource.edit(categories=categories) self._reload_variables() diff --git a/scrunch/helpers.py b/scrunch/helpers.py index e174e4f7..a89572dc 100644 --- a/scrunch/helpers.py +++ b/scrunch/helpers.py @@ -1,5 +1,6 @@ import requests import six +from datetime import datetime if six.PY2: # pragma: no cover from urlparse import urljoin @@ -232,3 +233,25 @@ def shoji_catalog_wrapper(index, **kwargs): } payload.update(**kwargs) return payload + + +def valid_categorical_date(date_str): + """ + Categories accept a `date` attribute that needs to be a valid ISO8601 date. + In order to keep dependencies reduced (no dateutil) and Python2x support, + we will support a limited set of simple date formats. + """ + valid_date_masks = [ + "%Y", + "%Y-%m", + "%Y-%m-%d", + ] + for mask in valid_date_masks: + try: + datetime.strptime(date_str, mask) + return True + except ValueError: + # Did not validate for this mask + continue + return False + diff --git a/scrunch/tests/test_datasets.py b/scrunch/tests/test_datasets.py index 9e182e43..95233d88 100644 --- a/scrunch/tests/test_datasets.py +++ b/scrunch/tests/test_datasets.py @@ -1437,6 +1437,36 @@ def test_add_category(self): var.add_category(2, 'New category', 2, before_id=9) var.resource._edit.assert_called_with(categories=var.resource.body['categories']) + def test_add_category_date(self): + ds_mock = self._dataset_mock() + ds = BaseDataset(ds_mock) + var = ds['var4_alias'] + var.resource.body['type'] = 'categorical' + categories = [ + {"id": 1, "name": "Female", "missing": False, "numeric_value": 1}, + {"id": 8, "name": "Male", "missing": False, "numeric_value": 8}, + {"id": 9, "name": "No Data", "missing": True, "numeric_value": 9} + ] + var.resource.body['categories'] = categories[:] + + with pytest.raises(ValueError) as err: + var.add_category(2, 'New category', 2, date=object()) + assert str(err.value) == "Date must be a string" + + with pytest.raises(ValueError) as err: + var.add_category(2, 'New category', 2, date="invalid date") + assert str(err.value) == "Date must conform to Y-m-d format" + + var.add_category(2, 'New category', 2, date="2021-12-12") + new_categories = categories[:] + [{ + "id": 2, + "name": "New category", + "date": "2021-12-12", + "numeric_value": 2, + "missing": False + }] + var.resource._edit.assert_called_with(categories=new_categories) + def test_integrate_variables(self): ds_mock = mock.MagicMock() var_tuple = mock.MagicMock() diff --git a/tox.ini b/tox.ini index 40d52b56..c7d2da9c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36,py37,{py27,py36,py37}-pandas +envlist = py27,py34,py36,py37,{py27,py36,py37}-pandas minversion = 2.4 skip_missing_interpreters = true @@ -7,7 +7,6 @@ skip_missing_interpreters = true python = 2.7: py27,py27-pandas 3.4: py34 - 3.5: py35,py35-pandas 3.6: py36,py36-pandas 3.7: py37,py37-pandas