Source Data Checks: Refactor column class for ingest & distribution

dcpy/connectors/socrata/metadata.py

@@ -54,7 +54,7 @@ def make_dcp_col(c: pub.Socrata.Responses.Column) -> md.DatasetColumn:
         dcp_col["values"] = [
             {"value": s["item"], "description": FILL_ME_IN_PLACEHOLDER} for s in samples
         ]
-    md.DatasetColumn._validate_data_type = False
+    # md.DatasetColumn._validate_data_type = False # legacy attribute used during migration, no longer there
     return md.DatasetColumn(**dcp_col)
 
 

dcpy/lifecycle/package/shapefiles.py

@@ -7,10 +7,11 @@
     DatasetAttributes,
     DatasetColumn,
     ColumnValue,
+    COLUMN_TYPES,
 )
 from dcpy.utils.logging import logger
 
-_shapefile_to_dcpy_types = {
+_shapefile_to_dcpy_types: dict[str, COLUMN_TYPES] = {
     "OID": "integer",
     "Integer": "integer",
     "SmallInteger": "integer",
@@ -19,7 +20,7 @@
     "String": "text",
     "Date": "datetime",
     "Geometry": "geometry",
-    "Boolean": "boolean",
+    "Boolean": "bool",
 }
 
 

dcpy/models/dataset.py

@@ -0,0 +1,32 @@
+from dcpy.models.base import SortedSerializedBase
+from typing import Literal
+
+COLUMN_TYPES = Literal[
+    "text",
+    "integer",
+    "decimal",
+    "number",  # TODO: Need to delete. Keeping it now for compatibility with metadata files
+    "geometry",
+    "bool",
+    "bbl",
+    "date",
+    "datetime",
+]
+
+
+# TODO: extend/modify Checks model
+class Checks(SortedSerializedBase):
+    is_primary_key: bool | None = None
+    non_nullable: bool | None = None
+
+
+class Column(SortedSerializedBase, extra="forbid"):
+    """
+    An extensible base class for defining column metadata in ingest and product templates.
+    """
+
+    id: str
+    data_type: COLUMN_TYPES | None = None
+    description: str | None = None
+    is_required: bool = True
+    checks: Checks | None = None

dcpy/models/lifecycle/ingest.py

@@ -9,6 +9,7 @@
 from dcpy.models.connectors import web, socrata
 from dcpy.models import file
 from dcpy.models.base import SortedSerializedBase
+from dcpy.models.dataset import Column as BaseColumn, COLUMN_TYPES
 
 
 class LocalFileSource(BaseModel, extra="forbid"):
@@ -77,12 +78,10 @@ class Ingestion(SortedSerializedBase):
     processing_steps: list[ProcessingStep] = []
 
 
-class Column(SortedSerializedBase):
-    id: str
-    data_type: Literal[
-        "text", "integer", "decimal", "geometry", "bool", "date", "datetime"
-    ]
-    description: str | None = None
+class Column(BaseColumn):
+    _head_sort_order = ["id", "data_type", "description"]
+
+    data_type: COLUMN_TYPES  # override BaseColumn `data_type` to be required field
 
 
 class Template(BaseModel, extra="forbid"):

dcpy/models/product/dataset/metadata_v2.py

@@ -1,12 +1,12 @@
 from __future__ import annotations
 
-from pydantic import field_validator
 from pydantic import BaseModel
-from typing import Any, List, Literal, get_args
+from typing import Any, List
 import unicodedata
 
 from dcpy.utils.collections import deep_merge_dict as merge
 from dcpy.models.base import SortedSerializedBase, YamlWriter, TemplatedYamlReader
+from dcpy.models.dataset import Column, COLUMN_TYPES
 
 ERROR_MISSING_COLUMN = "MISSING COLUMN"
 
@@ -48,50 +48,26 @@ class CustomizableBase(SortedSerializedBase, extra="forbid"):
 
 
 # COLUMNS
-# TODO: move to share with ingest.validate
-class Checks(CustomizableBase):
-    is_primary_key: bool | None = None
-    non_nullable: bool | None = None
-
-
-# TODO: move to share with ingest.validate
-COLUMN_TYPES = Literal[
-    "text", "number", "integer", "decimal", "geometry", "bool", "bbl", "datetime"
-]
-
-
 class ColumnValue(CustomizableBase):
     _head_sort_order = ["value", "description"]
 
     value: str
     description: str | None = None
 
 
-class DatasetColumn(CustomizableBase):
+class DatasetColumn(Column):
     _head_sort_order = ["id", "name", "data_type", "description"]
     _tail_sort_order = ["example", "values", "custom"]
-    _validate_data_type = (
-        True  # override, to generate md where we don't know the data_type
-    )
 
     # Note: id isn't intended to be overrideable, but is always required as a
-    # pointer back to the original column, so it is required here.
-    id: str
+    # pointer back to the original column.
     name: str | None = None
-    data_type: str | None = None
     data_source: str | None = None
-    description: str | None = None
     notes: str | None = None
     example: str | None = None
-    checks: Checks | None = None
     deprecated: bool | None = None
     values: list[ColumnValue] | None = None
-
-    @field_validator("data_type")
-    def _validate_colum_types(cls, v):
-        if cls._validate_data_type:
-            assert v in get_args(COLUMN_TYPES)
-        return v
+    custom: dict[str, Any] = {}
 
     def override(self, overrides: DatasetColumn) -> DatasetColumn:
         return DatasetColumn(**merge(self.model_dump(), overrides.model_dump()))
@@ -374,11 +350,13 @@ def validate_consistency(self):
         return errors
 
     def apply_column_defaults(
-        self, column_defaults: dict[tuple[str, str], DatasetColumn]
+        self, column_defaults: dict[tuple[str, COLUMN_TYPES], DatasetColumn]
     ) -> list[DatasetColumn]:
         return [
-            c.override(column_defaults[c.id, c.data_type])
-            if c.data_type and (c.id, c.data_type) in column_defaults
-            else c
+            (
+                c.override(column_defaults[c.id, c.data_type])
+                if c.data_type and (c.id, c.data_type) in column_defaults
+                else c
+            )
             for c in self.columns
         ]

dcpy/models/product/metadata.py

@@ -9,6 +9,7 @@
     Metadata as DatasetMetadata,
     DatasetColumn,
     DatasetOrgProductAttributesOverride,
+    COLUMN_TYPES,
 )
 from dcpy.utils.collections import deep_merge_dict as merge
 
@@ -39,15 +40,15 @@ class ProductMetadata(SortedSerializedBase, extra="forbid"):
     root_path: Path
     metadata: ProductMetadataFile
     template_vars: dict = {}
-    column_defaults: dict[tuple[str, str], DatasetColumn] = {}
+    column_defaults: dict[tuple[str, COLUMN_TYPES], DatasetColumn] = {}
     org_attributes: DatasetOrgProductAttributesOverride
 
     @classmethod
     def from_path(
         cls,
         root_path: Path,
         template_vars: dict = {},
-        column_defaults: dict[tuple[str, str], DatasetColumn] = {},
+        column_defaults: dict[tuple[str, COLUMN_TYPES], DatasetColumn] = {},
         org_attributes: DatasetOrgProductAttributesOverride = DatasetOrgProductAttributesOverride(),
     ) -> ProductMetadata:
         return ProductMetadata(
@@ -121,7 +122,7 @@ class OrgMetadata(SortedSerializedBase, extra="forbid"):
     root_path: Path
     template_vars: dict = Field(default_factory=dict)
     metadata: OrgMetadataFile
-    column_defaults: dict[tuple[str, str], DatasetColumn]
+    column_defaults: dict[tuple[str, COLUMN_TYPES], DatasetColumn]
 
     @classmethod
     def get_string_snippets(cls, path: Path) -> dict:
@@ -136,7 +137,9 @@ def get_string_snippets(cls, path: Path) -> dict:
         return yml
 
     @classmethod
-    def get_column_defaults(cls, path: Path) -> dict[tuple[str, str], DatasetColumn]:
+    def get_column_defaults(
+        cls, path: Path
+    ) -> dict[tuple[str, COLUMN_TYPES], DatasetColumn]:
         c_path = path / "snippets" / "column_defaults.yml"
         if not c_path.exists():
             return {}

dcpy/test/lifecycle/package/test_column_validation.py

@@ -63,8 +63,6 @@ def bbl(boro_code, block, lot):
 def _fake_row(columns: list[md.DatasetColumn]):
     row = {}
 
-    found_bbl_parts = {}
-    bbl_parts = {"boro_code", "block", "lot"}
     found_bbl_name = ""
     for c in columns:
         if c.data_type == "bbl":
@@ -74,23 +72,14 @@ def _fake_row(columns: list[md.DatasetColumn]):
         else:
             val = fakes[c.data_type or ""]()
             row[c.name] = val
-            if c.data_type in {"boro_code", "block", "lot"}:
-                found_bbl_parts[c.data_type] = val
 
-    # Construct a BBL from found parts, or generate a new one
+    # Generate a new bbl value
     if found_bbl_name:
-        if set(found_bbl_parts.keys()) == bbl_parts:
-            row[found_bbl_name] = fakes["bbl"](
-                found_bbl_parts["boro_code"],
-                found_bbl_parts["block"],
-                found_bbl_parts["lot"],
-            )
-        else:
-            row[found_bbl_name] = fakes["bbl"](
-                fakes["boro_code"](),
-                fakes["block"](),
-                fakes["lot"](),
-            )
+        row[found_bbl_name] = fakes["bbl"](
+            fakes["boro_code"](),
+            fakes["block"](),
+            fakes["lot"](),
+        )
 
     for c in columns:
         if c.checks and not c.checks.non_nullable and random.choice([True, False]):

dcpy/test/models/product/test_metadata.py

@@ -3,6 +3,7 @@
 
 from dcpy.models.product import metadata as md
 from dcpy.models.product.dataset import metadata_v2 as ds_md
+from dcpy.models import dataset
 
 
 @pytest.fixture
@@ -178,7 +179,7 @@ def test_column_defaults_applied(dataset_with_snippets: ds_md.Metadata):
             name="uid",
             data_type="text",
             data_source="Department of City Planning",
-            checks=ds_md.Checks(is_primary_key=True),
+            checks=dataset.Checks(is_primary_key=True),
         ),
         ds_md.DatasetColumn(
             id="bbl",