diff --git a/CHANGES.rst b/CHANGES.rst index 6f522f6..4d9f820 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -11,7 +11,6 @@ Changelog of threedi-schema - Make model_settings.use_2d_rain and model_settings.friction_averaging booleans - Remove columns referencing v2 in geometry_column - Ensure correct use_* values when matching tables have no data -- Use custom types for comma separated and table text fields to strip extra white space - Correct direction of dwf and surface map diff --git a/threedi_schema/domain/custom_types.py b/threedi_schema/domain/custom_types.py index b778373..cdff404 100644 --- a/threedi_schema/domain/custom_types.py +++ b/threedi_schema/domain/custom_types.py @@ -1,6 +1,6 @@ import geoalchemy2 from packaging import version -from sqlalchemy.types import Integer, Text, TypeDecorator, VARCHAR +from sqlalchemy.types import Integer, TypeDecorator, VARCHAR class Geometry(geoalchemy2.types.Geometry): @@ -66,46 +66,6 @@ class IntegerEnum(CustomEnum): impl = Integer -class CSVText(TypeDecorator): - impl = Text - cache_ok = True - - def process_bind_param(self, value, dialect): - if value is not None: - # custom clean up behavior - value = value.replace(" ", "").replace("\n", "") - return value - - def process_result_value(self, value, dialect): - if value is not None: - # custom clean up behavior - value = value.replace(" ", "").replace("\n", "") - return value - - -class CSVTable(TypeDecorator): - impl = Text - cache_ok = True - - def process_bind_param(self, value, dialect): - if value is not None: - # convert windows line endings to unix first - value = value.replace("\r\n", "\n") - # clean up each line - lines = value.split("\n") - cleaned_lines = [line.replace(" ", "") for line in lines if line] - value = "\n".join(cleaned_lines) - return value - - def process_result_value(self, value, dialect): - if value is not None: - # no need to replace \r\n here as the value came from the DB - lines = value.split("\n") - cleaned_lines = [line.replace(" ", "") for line in lines if line] - value = "\n".join(cleaned_lines) - return value - - class VarcharEnum(CustomEnum): cache_ok = True impl = VARCHAR diff --git a/threedi_schema/domain/models.py b/threedi_schema/domain/models.py index 61dd81e..ace8bc8 100644 --- a/threedi_schema/domain/models.py +++ b/threedi_schema/domain/models.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import declarative_base from . import constants -from .custom_types import CSVTable, CSVText, Geometry, IntegerEnum, VarcharEnum +from .custom_types import Geometry, IntegerEnum, VarcharEnum Base = declarative_base() # automap_base() @@ -13,12 +13,12 @@ class Lateral2D(Base): code = Column(Text) display_name = Column(Text) type = Column(IntegerEnum(constants.Later2dType)) - timeseries = Column(CSVText) + timeseries = Column(Text) time_units = Column(Text) interpolate = Column(Boolean) offset = Column(Integer) units = Column(Text) - tags = Column(CSVText) + tags = Column(Text) geom = Column(Geometry("POINT"), nullable=False) @@ -28,10 +28,10 @@ class BoundaryConditions2D(Base): code = Column(Text) display_name = Column(Text) type = Column(IntegerEnum(constants.BoundaryType)) - timeseries = Column(CSVText) + timeseries = Column(Text) time_units = Column(Text) interpolate = Column(Boolean) - tags = Column(CSVText) + tags = Column(Text) geom = Column(Geometry("LINESTRING"), nullable=False) @@ -43,7 +43,7 @@ class ControlMeasureLocation(Base): display_name = Column(Text) code = Column(Text) geom = Column(Geometry("POINT"), nullable=False) - tags = Column(CSVText) + tags = Column(Text) class ControlMeasureMap(Base): @@ -56,7 +56,7 @@ class ControlMeasureMap(Base): display_name = Column(Text) code = Column(Text) geom = Column(Geometry("LINESTRING"), nullable=False) - tags = Column(CSVText) + tags = Column(Text) class ControlMemory(Base): @@ -74,13 +74,13 @@ class ControlMemory(Base): display_name = Column(Text) code = Column(Text) geom = Column(Geometry("POINT"), nullable=False) - tags = Column(CSVText) + tags = Column(Text) class ControlTable(Base): __tablename__ = "table_control" id = Column(Integer, primary_key=True) - action_table = Column(CSVTable) + action_table = Column(Text) action_type = Column(VarcharEnum(constants.ControlTableActionTypes)) measure_operator = Column(VarcharEnum(constants.MeasureOperators)) target_type = Column(VarcharEnum(constants.StructureControlTypes)) @@ -88,7 +88,7 @@ class ControlTable(Base): display_name = Column(Text) code = Column(Text) geom = Column(Geometry("POINT"), nullable=False) - tags = Column(CSVText) + tags = Column(Text) class Interflow(Base): @@ -130,7 +130,7 @@ class SurfaceParameter(Base): min_infiltration_capacity = Column(Float, nullable=False) infiltration_decay_constant = Column(Float, nullable=False) infiltration_recovery_constant = Column(Float, nullable=False) - tags = Column(CSVText) + tags = Column(Text) description = Column(Text) @@ -145,7 +145,7 @@ class Surface(Base): Geometry("POLYGON"), nullable=True, ) - tags = Column(CSVText) + tags = Column(Text) class DryWeatherFlow(Base): @@ -161,7 +161,7 @@ class DryWeatherFlow(Base): Geometry("POLYGON"), nullable=False, ) - tags = Column(CSVText) + tags = Column(Text) class DryWeatherFlowMap(Base): @@ -176,15 +176,15 @@ class DryWeatherFlowMap(Base): nullable=False, ) percentage = Column(Float) - tags = Column(CSVText) + tags = Column(Text) class DryWeatherFlowDistribution(Base): __tablename__ = "dry_weather_flow_distribution" id = Column(Integer, primary_key=True) description = Column(Text) - tags = Column(CSVText) - distribution = Column(CSVText) + tags = Column(Text) + distribution = Column(Text) class GroundWater(Base): @@ -237,7 +237,7 @@ class GridRefinementLine(Base): grid_level = Column(Integer) geom = Column(Geometry("LINESTRING"), nullable=False) code = Column(String(100)) - tags = Column(CSVText) + tags = Column(Text) class GridRefinementArea(Base): @@ -247,7 +247,7 @@ class GridRefinementArea(Base): grid_level = Column(Integer) code = Column(String(100)) geom = Column(Geometry("POLYGON"), nullable=False) - tags = Column(CSVText) + tags = Column(Text) class ConnectionNode(Base): @@ -255,7 +255,7 @@ class ConnectionNode(Base): id = Column(Integer, primary_key=True) geom = Column(Geometry("POINT"), nullable=False) code = Column(String(100)) - tags = Column(CSVText) + tags = Column(Text) display_name = Column(Text) storage_area = Column(Float) initial_water_level = Column(Float) @@ -274,12 +274,12 @@ class Lateral1d(Base): id = Column(Integer, primary_key=True) code = Column(Text) display_name = Column(Text) - timeseries = Column(CSVText) + timeseries = Column(Text) time_units = Column(Text) interpolate = Column(Boolean) offset = Column(Integer) units = Column(Text) - tags = Column(CSVText) + tags = Column(Text) geom = Column(Geometry("POINT"), nullable=False) connection_node_id = Column(Integer) @@ -435,10 +435,10 @@ class BoundaryCondition1D(Base): code = Column(Text) display_name = Column(Text) type = Column(IntegerEnum(constants.BoundaryType)) - timeseries = Column(CSVText) + timeseries = Column(Text) time_units = Column(Text) interpolate = Column(Boolean) - tags = Column(CSVText) + tags = Column(Text) geom = Column(Geometry("POINT"), nullable=False) connection_node_id = Column(Integer) @@ -451,7 +451,7 @@ class SurfaceMap(Base): connection_node_id = Column(Integer) percentage = Column(Float) geom = Column(Geometry("LINESTRING"), nullable=False) - tags = Column(CSVText) + tags = Column(Text) code = Column(String(100)) display_name = Column(String(255)) @@ -461,7 +461,7 @@ class Channel(Base): id = Column(Integer, primary_key=True) display_name = Column(String(255)) code = Column(String(100)) - tags = Column(CSVText) + tags = Column(Text) exchange_type = Column(IntegerEnum(constants.CalculationType)) calculation_point_distance = Column(Float) geom = Column(Geometry("LINESTRING"), nullable=False) @@ -485,14 +485,14 @@ class Windshielding(Base): northwest = Column(Float) geom = Column(Geometry("POINT"), nullable=False) channel_id = Column(Integer) - tags = Column(CSVText) + tags = Column(Text) class CrossSectionLocation(Base): __tablename__ = "cross_section_location" id = Column(Integer, primary_key=True) code = Column(String(100)) - tags = Column(CSVText) + tags = Column(Text) reference_level = Column(Float) friction_type = Column(IntegerEnum(constants.FrictionType)) friction_value = Column(Float) @@ -500,9 +500,9 @@ class CrossSectionLocation(Base): cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) cross_section_width = Column(Float) cross_section_height = Column(Float) - cross_section_friction_values = Column(CSVText) - cross_section_vegetation_table = Column(CSVTable) - cross_section_table = Column(CSVTable) + cross_section_friction_values = Column(Text) + cross_section_vegetation_table = Column(Text) + cross_section_table = Column(Text) vegetation_stem_density = Column(Float) vegetation_stem_diameter = Column(Float) vegetation_height = Column(Float) @@ -516,7 +516,7 @@ class Pipe(Base): id = Column(Integer, primary_key=True) display_name = Column(String(255)) code = Column(String(100)) - tags = Column(CSVText) + tags = Column(Text) geom = Column(Geometry("LINESTRING"), nullable=False) sewerage_type = Column(IntegerEnum(constants.SewerageType)) exchange_type = Column(IntegerEnum(constants.PipeCalculationType)) @@ -531,7 +531,7 @@ class Pipe(Base): cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) cross_section_width = Column(Float) cross_section_height = Column(Float) - cross_section_table = Column(CSVTable) + cross_section_table = Column(Text) exchange_thickness = Column(Float) hydraulic_conductivity_in = Column(Float) hydraulic_conductivity_out = Column(Float) @@ -542,7 +542,7 @@ class Culvert(Base): id = Column(Integer, primary_key=True) display_name = Column(String(255)) code = Column(String(100)) - tags = Column(CSVText) + tags = Column(Text) exchange_type = Column(IntegerEnum(constants.CalculationTypeCulvert)) friction_value = Column(Float) friction_type = Column(IntegerEnum(constants.FrictionType)) @@ -558,7 +558,7 @@ class Culvert(Base): cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) cross_section_width = Column(Float) cross_section_height = Column(Float) - cross_section_table = Column(CSVTable) + cross_section_table = Column(Text) class DemAverageArea(Base): @@ -567,7 +567,7 @@ class DemAverageArea(Base): geom = Column(Geometry("POLYGON"), nullable=False) display_name = Column(Text) code = Column(Text) - tags = Column(CSVText) + tags = Column(Text) class Weir(Base): @@ -576,7 +576,7 @@ class Weir(Base): code = Column(String(100)) display_name = Column(String(255)) geom = Column(Geometry("LINESTRING"), nullable=False) - tags = Column(CSVText) + tags = Column(Text) crest_level = Column(Float) crest_type = Column(IntegerEnum(constants.CrestType)) friction_value = Column(Float) @@ -591,7 +591,7 @@ class Weir(Base): cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) cross_section_width = Column(Float) cross_section_height = Column(Float) - cross_section_table = Column(CSVTable) + cross_section_table = Column(Text) class Orifice(Base): @@ -599,7 +599,7 @@ class Orifice(Base): id = Column(Integer, primary_key=True) code = Column(String(100)) display_name = Column(String(255)) - tags = Column(CSVText) + tags = Column(Text) geom = Column(Geometry("LINESTRING"), nullable=False) crest_type = Column(IntegerEnum(constants.CrestType)) crest_level = Column(Float) @@ -614,7 +614,7 @@ class Orifice(Base): cross_section_shape = Column(IntegerEnum(constants.CrossSectionShape)) cross_section_width = Column(Float) cross_section_height = Column(Float) - cross_section_table = Column(CSVTable) + cross_section_table = Column(Text) class Pump(Base): @@ -632,7 +632,7 @@ class Pump(Base): sewerage = Column(Boolean) connection_node_id = Column(Integer) geom = Column(Geometry("POINT"), nullable=False) - tags = Column(CSVText) + tags = Column(Text) class PumpMap(Base): @@ -641,7 +641,7 @@ class PumpMap(Base): pump_id = Column(Integer) connection_node_id_end = Column(Integer) geom = Column(Geometry("LINESTRING"), nullable=False) - tags = Column(CSVText) + tags = Column(Text) code = Column(String(100)) display_name = Column(String(255)) @@ -652,7 +652,7 @@ class Obstacle(Base): code = Column(String(100)) crest_level = Column(Float) geom = Column(Geometry("LINESTRING"), nullable=False) - tags = Column(CSVText) + tags = Column(Text) display_name = Column(String(255)) affects_2d = Column(Boolean) affects_1d2d_open_water = Column(Boolean) @@ -664,7 +664,7 @@ class PotentialBreach(Base): id = Column(Integer, primary_key=True) code = Column(String(100)) display_name = Column(String(255)) - tags = Column(CSVText) + tags = Column(Text) initial_exchange_level = Column(Float) final_exchange_level = Column(Float) levee_material = Column(IntegerEnum(constants.Material)) @@ -680,7 +680,7 @@ class ExchangeLine(Base): exchange_level = Column(Float) display_name = Column(Text) code = Column(Text) - tags = Column(CSVText) + tags = Column(Text) class Tags(Base):