From e290bfb320ef8e6637ab011848b48f629cf42b1e Mon Sep 17 00:00:00 2001 From: kirk Date: Thu, 24 Oct 2024 17:32:51 -0400 Subject: [PATCH 1/6] remove enforcement --- src/fideslang/models.py | 29 --------------------------- tests/fideslang/test_validation.py | 32 +++++++++++++----------------- 2 files changed, 14 insertions(+), 47 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index bb15f25..0b12b22 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -471,35 +471,6 @@ def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]: ) return meta_values - @model_validator(mode="after") - def validate_object_fields( - self, - _: ValidationInfo, - ) -> DatasetField: - """Two validation checks for object fields: - - If there are sub-fields specified, type should be either empty or 'object' - - Additionally object fields cannot have data_categories. - """ - fields = self.fields - declared_data_type = None - field_name: str = self.name - - if self.fides_meta: - declared_data_type = self.fides_meta.data_type - - if fields and declared_data_type: - data_type, _ = parse_data_type_string(declared_data_type) - if data_type != "object": - raise ValueError( - f"The data type '{data_type}' on field '{field_name}' is not compatible with specified sub-fields. Convert to an 'object' field." - ) - - if (fields or declared_data_type == "object") and self.data_categories: - raise ValueError( - f"Object field '{field_name}' cannot have specified data_categories. Specify category on sub-field instead" - ) - return self - # this is required for the recursive reference in the pydantic model: DatasetField.model_rebuild() diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index d14c145..7c0fdcd 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -717,23 +717,20 @@ def test_return_all_elements_on_array_field(self): ) def test_data_categories_at_object_level(self): - with pytest.raises(ValidationError) as exc: - DatasetField( - name="test_field", - data_categories=["user"], - fides_meta=FidesMeta( - references=None, - identify=None, - primary_key=False, - data_type="object", - length=None, - return_all_elements=None, - read_only=None, - ), - fields=[DatasetField(name="nested_field")], - ) - assert_error_message_includes( - exc, "Object field 'test_field' cannot have specified data_categories" + # Data categories at the object level ARE allowed now + DatasetField( + name="test_field", + data_categories=["user"], + fides_meta=FidesMeta( + references=None, + identity=None, + primary_key=False, + data_type="object", + length=None, + return_all_elements=None, + read_only=None, + ), + fields=[DatasetField(name="nested_field")], ) def test_object_field_conflicting_types(self): @@ -806,7 +803,6 @@ def test_erase_after(self): assert meta.erase_after == [FidesCollectionKey("test_dataset.test_collection")] - class TestAnyUrlString: def test_valid_url(self): assert AnyUrlString("https://www.example.com/") From 6443566b8fee6c890ca55a25c25ea4b29526ae7a Mon Sep 17 00:00:00 2001 From: kirk Date: Tue, 29 Oct 2024 16:14:31 -0400 Subject: [PATCH 2/6] test update --- tests/fideslang/test_validation.py | 31 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index 7c0fdcd..d5521df 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -734,23 +734,20 @@ def test_data_categories_at_object_level(self): ) def test_object_field_conflicting_types(self): - with pytest.raises(ValidationError) as exc: - DatasetField( - name="test_field", - data_categories=["user"], - fides_meta=FidesMeta( - references=None, - identify=None, - primary_key=False, - data_type="string", - length=None, - return_all_elements=None, - read_only=None, - ), - fields=[DatasetField(name="nested_field")], - ) - assert_error_message_includes( - exc, "The data type 'string' on field 'test_field' is not compatible with" + + DatasetField( + name="test_field", + data_categories=["user"], + fides_meta=FidesMeta( + references=None, + identify=None, + primary_key=False, + data_type="string", + length=None, + return_all_elements=None, + read_only=None, + ), + fields=[DatasetField(name="nested_field")], ) def test_data_categories_on_nested_fields(self): From 81e96eb2d97508661c8cb97df9fcdcd54a4b77b9 Mon Sep 17 00:00:00 2001 From: kirk Date: Mon, 4 Nov 2024 10:18:57 -0500 Subject: [PATCH 3/6] test changes --- CHANGELOG.md | 4 +++ src/fideslang/models.py | 2 -- tests/conftest.py | 1 + tests/fideslang/test_validation.py | 47 +++++++++++++++++------------- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c5311f..e23bef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ The types of changes are: ## [Unreleased](https://github.com/ethyca/fideslang/compare/3.0.7...main) +### Changed + +- Remove stipulation that sub fields (Field) of a Field object cannot have data categories assigned [#22](https://github.com/ethyca/fideslang/pull/22) + ## [3.0.7](https://github.com/ethyca/fideslang/compare/3.0.6...3.0.7) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 0b12b22..21b9b14 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -18,7 +18,6 @@ HttpUrl, PositiveInt, SerializeAsAny, - ValidationInfo, field_validator, model_validator, ) @@ -32,7 +31,6 @@ is_deprecated_if_replaced, matching_parent_key, no_self_reference, - parse_data_type_string, sort_list_objects_by_name, unique_items_in_list, valid_data_type, diff --git a/tests/conftest.py b/tests/conftest.py index 2f177e9..142c021 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ """Common fixtures to be used across tests.""" + import os from typing import Any, Dict diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index d5521df..21f3174 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -729,40 +729,45 @@ def test_data_categories_at_object_level(self): length=None, return_all_elements=None, read_only=None, - ), - fields=[DatasetField(name="nested_field")], + ) ) - def test_object_field_conflicting_types(self): - - DatasetField( - name="test_field", + def test_data_categories_on_nested_fields(self): + + field = DatasetField( + name="test_for_nest", data_categories=["user"], fides_meta=FidesMeta( references=None, identify=None, primary_key=False, - data_type="string", + data_type="object", length=None, return_all_elements=None, read_only=None, ), - fields=[DatasetField(name="nested_field")], + fields=[ + DatasetField( + name="nested_field", + data_categories=["user"], + fides_meta=FidesMeta( + references=None, + identify=None, + primary_key=False, + data_type="string", + length=None, + return_all_elements=None, + read_only=None, + ), + ) + ], ) - def test_data_categories_on_nested_fields(self): - DatasetField( - name="test_field", - fides_meta=FidesMeta( - references=None, - identify=None, - primary_key=False, - data_type="object", - length=None, - read_only=None, - ), - fields=[DatasetField(name="nested_field", data_categories=["user"])], - ) + assert field + assert field.fields + print(f"fields: {field.fields}") + print(f"field: {vars(field)}") + assert field.fields[0].data_categories == ["user"] class TestCollectionMeta: From a0ef7a163c91661cd02f7faa579d26a9b734075c Mon Sep 17 00:00:00 2001 From: kirk Date: Mon, 4 Nov 2024 10:19:05 -0500 Subject: [PATCH 4/6] add back in validator --- src/fideslang/models.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 21b9b14..0a260de 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -18,6 +18,7 @@ HttpUrl, PositiveInt, SerializeAsAny, + ValidationInfo, field_validator, model_validator, ) @@ -31,6 +32,7 @@ is_deprecated_if_replaced, matching_parent_key, no_self_reference, + parse_data_type_string, sort_list_objects_by_name, unique_items_in_list, valid_data_type, @@ -468,6 +470,30 @@ def valid_meta(cls, meta_values: Optional[FidesMeta]) -> Optional[FidesMeta]: "The 'return_all_elements' attribute can only be specified on array fields." ) return meta_values + + @model_validator(mode="after") + def validate_object_fields( + self, + _: ValidationInfo, + ) -> DatasetField: + """Two validation checks for object fields: + - If there are sub-fields specified, type should be either empty or 'object' + """ + if self.fields: + breakpoint() + fields = self.fields + declared_data_type = None + field_name: str = self.name + + if self.fides_meta: + declared_data_type = self.fides_meta.data_type + + if fields and declared_data_type: + data_type, _ = parse_data_type_string(declared_data_type) + if data_type != "object": + raise ValueError( + f"The data type '{data_type}' on field '{field_name}' is not compatible with specified sub-fields. Convert to an 'object' field." + ) # this is required for the recursive reference in the pydantic model: From a445b8bbed963d2505f42eb3d63f2361a326fed2 Mon Sep 17 00:00:00 2001 From: kirk Date: Mon, 4 Nov 2024 10:20:25 -0500 Subject: [PATCH 5/6] remove breakpoint --- src/fideslang/models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index 0a260de..03e2979 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -479,8 +479,6 @@ def validate_object_fields( """Two validation checks for object fields: - If there are sub-fields specified, type should be either empty or 'object' """ - if self.fields: - breakpoint() fields = self.fields declared_data_type = None field_name: str = self.name From f19d678bcf9c9424c2c2cb966f502a807031debd Mon Sep 17 00:00:00 2001 From: kirk Date: Mon, 4 Nov 2024 13:45:29 -0500 Subject: [PATCH 6/6] real stuff --- src/fideslang/models.py | 4 -- tests/fideslang/test_validation.py | 82 ++++++++++++++++-------------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/fideslang/models.py b/src/fideslang/models.py index bb15f25..834b16b 100644 --- a/src/fideslang/models.py +++ b/src/fideslang/models.py @@ -494,10 +494,6 @@ def validate_object_fields( f"The data type '{data_type}' on field '{field_name}' is not compatible with specified sub-fields. Convert to an 'object' field." ) - if (fields or declared_data_type == "object") and self.data_categories: - raise ValueError( - f"Object field '{field_name}' cannot have specified data_categories. Specify category on sub-field instead" - ) return self diff --git a/tests/fideslang/test_validation.py b/tests/fideslang/test_validation.py index d14c145..1e9a672 100644 --- a/tests/fideslang/test_validation.py +++ b/tests/fideslang/test_validation.py @@ -717,59 +717,63 @@ def test_return_all_elements_on_array_field(self): ) def test_data_categories_at_object_level(self): - with pytest.raises(ValidationError) as exc: - DatasetField( - name="test_field", - data_categories=["user"], - fides_meta=FidesMeta( - references=None, - identify=None, - primary_key=False, - data_type="object", - length=None, - return_all_elements=None, - read_only=None, - ), - fields=[DatasetField(name="nested_field")], - ) - assert_error_message_includes( - exc, "Object field 'test_field' cannot have specified data_categories" - ) - def test_object_field_conflicting_types(self): - with pytest.raises(ValidationError) as exc: - DatasetField( - name="test_field", - data_categories=["user"], - fides_meta=FidesMeta( - references=None, - identify=None, - primary_key=False, - data_type="string", - length=None, - return_all_elements=None, - read_only=None, - ), - fields=[DatasetField(name="nested_field")], - ) - assert_error_message_includes( - exc, "The data type 'string' on field 'test_field' is not compatible with" + field = DatasetField( + name="test_field", + data_categories=["user"], + fides_meta=FidesMeta( + references=None, + identify=None, + primary_key=False, + data_type="object", + length=None, + return_all_elements=None, + read_only=None, + ), + fields=[DatasetField(name="nested_field")], ) + assert field + assert field.data_categories == ["user"] + assert field.fides_meta.data_type == "object" + + def test_data_categories_on_nested_fields(self): - DatasetField( - name="test_field", + + field = DatasetField( + name="test_for_nest", + data_categories=["user"], fides_meta=FidesMeta( references=None, identify=None, primary_key=False, data_type="object", length=None, + return_all_elements=None, read_only=None, ), - fields=[DatasetField(name="nested_field", data_categories=["user"])], + fields=[ + DatasetField( + name="nested_field", + data_categories=["user"], + fides_meta=FidesMeta( + references=None, + identify=None, + primary_key=False, + data_type="string", + length=None, + read_only=None, + data_categories=["user"], + ), + ) + ], ) + assert field + assert field.data_categories == ["user"] + assert field.fields[0].data_categories == ["user"] + + class TestCollectionMeta: def test_invalid_collection_key(self):