From 902d95fc4326f6fb91e0b855a5f12135c6b4349e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 19 Jul 2024 00:21:51 +0700 Subject: [PATCH 01/47] refactor: sync arg name Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/storage.py | 12 ++++++------ flexmeasures/data/schemas/sensors.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 58d169c1a..ab052b5e0 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -72,18 +72,18 @@ class StorageFlexModelSchema(Schema): # Timezone placeholder for the soc_maxima, soc_minima and soc_targets fields are overridden in __init__ soc_maxima = TimeSeriesOrSensor( - unit="MWh", timezone="placeholder", data_key="soc-maxima" + to_unit="MWh", timezone="placeholder", data_key="soc-maxima" ) soc_minima = TimeSeriesOrSensor( - unit="MWh", + to_unit="MWh", timezone="placeholder", data_key="soc-minima", value_validator=validate.Range(min=0), ) soc_targets = TimeSeriesOrSensor( - unit="MWh", timezone="placeholder", data_key="soc-targets" + to_unit="MWh", timezone="placeholder", data_key="soc-targets" ) soc_unit = fields.Str( @@ -122,17 +122,17 @@ def __init__(self, start: datetime, sensor: Sensor, *args, **kwargs): self.start = start self.sensor = sensor self.soc_maxima = TimeSeriesOrSensor( - unit="MWh", timezone=sensor.timezone, data_key="soc-maxima" + to_unit="MWh", timezone=sensor.timezone, data_key="soc-maxima" ) self.soc_minima = TimeSeriesOrSensor( - unit="MWh", + to_unit="MWh", timezone=sensor.timezone, data_key="soc-minima", value_validator=validate.Range(min=0), ) self.soc_targets = TimeSeriesOrSensor( - unit="MWh", timezone=sensor.timezone, data_key="soc-targets" + to_unit="MWh", timezone=sensor.timezone, data_key="soc-targets" ) super().__init__(*args, **kwargs) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 0c093b6e6..0882d74d8 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -289,7 +289,7 @@ def convert(self, value, param, ctx, **kwargs): class TimeSeriesOrSensor(MarshmallowClickMixin, fields.Field): def __init__( self, - unit, + to_unit, *args, timezone: str | None = None, value_validator: Validator | None = None, @@ -302,7 +302,7 @@ def __init__( super().__init__(*args, **kwargs) self.timezone = timezone self.value_validator = value_validator - self.unit = ur.Quantity(unit) + self.unit = ur.Quantity(to_unit) @with_appcontext_if_needed() def _deserialize( From 6e868d8d3d5e05730145b3929bba724997b64c14 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 19 Jul 2024 00:23:11 +0700 Subject: [PATCH 02/47] refactor: sync internal property name Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 0882d74d8..8a8a9c703 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -302,7 +302,7 @@ def __init__( super().__init__(*args, **kwargs) self.timezone = timezone self.value_validator = value_validator - self.unit = ur.Quantity(to_unit) + self.to_unit = ur.Quantity(to_unit) @with_appcontext_if_needed() def _deserialize( @@ -315,7 +315,7 @@ def _deserialize( "Dictionary provided but `sensor` key not found." ) - sensor = SensorIdField(unit=self.unit)._deserialize( + sensor = SensorIdField(unit=self.to_unit)._deserialize( value["sensor"], None, None ) From d8b2499700b47770e55997c6938b20701e5ea756 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 19 Jul 2024 00:28:53 +0700 Subject: [PATCH 03/47] refactor: flatten elif block Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 8a8a9c703..9ee777c49 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -249,12 +249,11 @@ def _deserialize( raise FMValidationError( f"Cannot convert value `{value}` to '{self.to_unit}'" ) from e + elif self.default_src_unit is not None: + return self._deserialize( + f"{value} {self.default_src_unit}", attr, obj, **kwargs + ) else: - if self.default_src_unit is not None: - return self._deserialize( - f"{value} {self.default_src_unit}", attr, obj, **kwargs - ) - raise FMValidationError( f"Unsupported value type. `{type(value)}` was provided but only dict and str are supported." ) From 77de0df6062c2ad36177305fb5a215966e86707d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 19 Jul 2024 00:31:42 +0700 Subject: [PATCH 04/47] refactor: allow string deserialization and allow setting a default source unit for interpreting values without a unit Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 9ee777c49..469d71117 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -290,6 +290,7 @@ def __init__( self, to_unit, *args, + default_src_unit: str | None = None, timezone: str | None = None, value_validator: Validator | None = None, **kwargs, @@ -302,6 +303,7 @@ def __init__( self.timezone = timezone self.value_validator = value_validator self.to_unit = ur.Quantity(to_unit) + self.default_src_unit = default_src_unit @with_appcontext_if_needed() def _deserialize( @@ -330,6 +332,20 @@ def _deserialize( ) return field._deserialize(value, None, None) + + elif isinstance(value, str): + try: + return ur.Quantity(value).to(self.to_unit) + except DimensionalityError as e: + raise FMValidationError( + f"Cannot convert value `{value}` to '{self.to_unit}'" + ) from e + + elif self.default_src_unit is not None: + return self._deserialize( + f"{value} {self.default_src_unit}", attr, obj, **kwargs + ) + else: raise FMValidationError( f"Unsupported value type. `{type(value)}` was provided but only dict and list are supported." From 8c9fe3f65d988f939365d3b552236788fc077edf Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 17:28:00 +0700 Subject: [PATCH 05/47] refactor: copy convert method Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 469d71117..6394b61f0 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -351,6 +351,20 @@ def _deserialize( f"Unsupported value type. `{type(value)}` was provided but only dict and list are supported." ) + def convert(self, value, param, ctx, **kwargs): + # case that the click default is defined in numeric values + if not isinstance(value, str): + return super().convert(value, param, ctx, **kwargs) + + _value = re.match(r"sensor:(\d+)", value) + + if _value is not None: + _value = {"sensor": int(_value.groups()[0])} + else: + _value = value + + return super().convert(_value, param, ctx, **kwargs) + class RepurposeValidatorToIgnoreSensors(validate.Validator): """Validator that executes another validator (the one you initialize it with) only on non-Sensor values.""" From ef0d5e89911891103ace4f195f5e31bd204f4eaf Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 17:29:33 +0700 Subject: [PATCH 06/47] fix: type annotation Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 6394b61f0..27e721ef1 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -259,7 +259,7 @@ def _deserialize( ) def _serialize( - self, value: ur.Quantity | dict[str, Sensor], attr, data, **kwargs + self, value: ur.Quantity | Sensor, attr, data, **kwargs ) -> str | dict[str, int]: if isinstance(value, ur.Quantity): return str(value.to(self.to_unit)) From a9364b1a8de4593b422f78bbf359ac8d424452ac Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 17:30:20 +0700 Subject: [PATCH 07/47] refactor: copy _serialize method Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 27e721ef1..d7c13bf3b 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -351,6 +351,18 @@ def _deserialize( f"Unsupported value type. `{type(value)}` was provided but only dict and list are supported." ) + def _serialize( + self, value: ur.Quantity | Sensor, attr, data, **kwargs + ) -> str | dict[str, int]: + if isinstance(value, ur.Quantity): + return str(value.to(self.to_unit)) + elif isinstance(value, Sensor): + return dict(sensor=value.id) + else: + raise FMValidationError( + "Serialized Quantity Or Sensor needs to be of type int, float or Sensor" + ) + def convert(self, value, param, ctx, **kwargs): # case that the click default is defined in numeric values if not isinstance(value, str): From b3070f726ca4d43a274ac175d56cd7c367ab724e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 17:32:44 +0700 Subject: [PATCH 08/47] refactor: adapt _serialize method Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index d7c13bf3b..46e372e7d 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -352,15 +352,19 @@ def _deserialize( ) def _serialize( - self, value: ur.Quantity | Sensor, attr, data, **kwargs + self, value: ur.Quantity | Sensor | pd.Series, attr, data, **kwargs ) -> str | dict[str, int]: if isinstance(value, ur.Quantity): return str(value.to(self.to_unit)) elif isinstance(value, Sensor): return dict(sensor=value.id) + elif isinstance(value, pd.Series): + raise NotImplementedError( + "Serialization of a time series from a Pandas Series is not implemented yet." + ) else: raise FMValidationError( - "Serialized Quantity Or Sensor needs to be of type int, float or Sensor" + "Serialized quantity, sensor or time series needs to be of type int, float, Sensor or pandas.Series." ) def convert(self, value, param, ctx, **kwargs): From 1339b1ad61e0e18cad5d6fdec6eccf3c836421c0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 17:39:40 +0700 Subject: [PATCH 09/47] refactor: adapt docstring Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 46e372e7d..5a5d3cf6f 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -295,9 +295,16 @@ def __init__( value_validator: Validator | None = None, **kwargs, ): - """ - The timezone is only used in case a time series is specified and one - of the *timed events* in the time series uses a nominal duration, such as "P1D". + """Field for validating, serializing and deserializing a quantity, sensor or time series. + + NB any validators passed are only applied to Quantities. + For example, validate=validate.Range(min=0) will raise a ValidationError in case of negative quantities, + but will let pass any sensor that has recorded negative values. + + :param to_unit: Unit in which the sensor or quantity should be convertible to. + :param default_src_unit: What unit to use in case of getting a numeric value. + :param timezone: Only used in case a time series is specified and one of the *timed events* + in the time series uses a nominal duration, such as "P1D". """ super().__init__(*args, **kwargs) self.timezone = timezone From 51070267c06ce30278a3dc62194dfa00e5acf096 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 17:43:43 +0700 Subject: [PATCH 10/47] refactor: update error message Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 5a5d3cf6f..9583d73d8 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -355,7 +355,7 @@ def _deserialize( else: raise FMValidationError( - f"Unsupported value type. `{type(value)}` was provided but only dict and list are supported." + f"Unsupported value type. `{type(value)}` was provided but only dict, list and str are supported." ) def _serialize( From 9084e53a5860550148442254b64dc13d2c7e77f5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 17:47:32 +0700 Subject: [PATCH 11/47] refactor: update type annotation Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 9583d73d8..8ee33b411 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -314,8 +314,8 @@ def __init__( @with_appcontext_if_needed() def _deserialize( - self, value: str | dict[str, int], attr, obj, **kwargs - ) -> list[dict] | Sensor: + self, value: dict[str, int] | list[dict] | str, attr, obj, **kwargs + ) -> list[dict] | Sensor | ur.Quantity: if isinstance(value, dict): if "sensor" not in value: From bb157cf6bacef079bd4e3055ba949f6961ee3264 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 18:13:20 +0700 Subject: [PATCH 12/47] refactor: merge schemas Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 26 +++--- flexmeasures/data/schemas/__init__.py | 2 +- .../data/schemas/scheduling/__init__.py | 8 +- .../data/schemas/scheduling/storage.py | 30 +++---- flexmeasures/data/schemas/sensors.py | 88 ++----------------- .../data/schemas/tests/test_sensor.py | 8 +- 6 files changed, 43 insertions(+), 119 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 7f1ac3e3c..5f17591da 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -62,7 +62,7 @@ LongitudeField, SensorIdField, TimeIntervalField, - QuantityOrSensor, + TimeSeriesOrQuantityOrSensor, ) from flexmeasures.data.schemas.sources import DataSourceIdField from flexmeasures.data.schemas.times import TimeIntervalSchema @@ -1123,7 +1123,7 @@ def create_schedule(ctx): @click.option( "--site-power-capacity", "site_power_capacity", - type=QuantityOrSensor("MW"), + type=TimeSeriesOrQuantityOrSensor("MW"), required=False, default=None, help="Site consumption/production power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1133,7 +1133,7 @@ def create_schedule(ctx): @click.option( "--site-consumption-capacity", "site_consumption_capacity", - type=QuantityOrSensor("MW"), + type=TimeSeriesOrQuantityOrSensor("MW"), required=False, default=None, help="Site consumption power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1143,7 +1143,7 @@ def create_schedule(ctx): @click.option( "--site-production-capacity", "site_production_capacity", - type=QuantityOrSensor("MW"), + type=TimeSeriesOrQuantityOrSensor("MW"), required=False, default=None, help="Site production power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1208,7 +1208,7 @@ def create_schedule(ctx): @click.option( "--charging-efficiency", "charging_efficiency", - type=QuantityOrSensor("%"), + type=TimeSeriesOrQuantityOrSensor("%"), required=False, default=None, help="Storage charging efficiency to use for the schedule." @@ -1218,7 +1218,7 @@ def create_schedule(ctx): @click.option( "--discharging-efficiency", "discharging_efficiency", - type=QuantityOrSensor("%"), + type=TimeSeriesOrQuantityOrSensor("%"), required=False, default=None, help="Storage discharging efficiency to use for the schedule." @@ -1228,7 +1228,7 @@ def create_schedule(ctx): @click.option( "--soc-gain", "soc_gain", - type=QuantityOrSensor("MW"), + type=TimeSeriesOrQuantityOrSensor("MW"), required=False, default=None, help="Specify the State of Charge (SoC) gain as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1238,7 +1238,7 @@ def create_schedule(ctx): @click.option( "--soc-usage", "soc_usage", - type=QuantityOrSensor("MW"), + type=TimeSeriesOrQuantityOrSensor("MW"), required=False, default=None, help="Specify the State of Charge (SoC) usage as a quantity in power units (e.g. 1 MW or 1000 kW) " @@ -1248,7 +1248,7 @@ def create_schedule(ctx): @click.option( "--storage-power-capacity", "storage_power_capacity", - type=QuantityOrSensor("MW"), + type=TimeSeriesOrQuantityOrSensor("MW"), required=False, default=None, help="Storage consumption/production power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1258,7 +1258,7 @@ def create_schedule(ctx): @click.option( "--storage-consumption-capacity", "storage_consumption_capacity", - type=QuantityOrSensor("MW"), + type=TimeSeriesOrQuantityOrSensor("MW"), required=False, default=None, help="Storage consumption power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1268,7 +1268,7 @@ def create_schedule(ctx): @click.option( "--storage-production-capacity", "storage_production_capacity", - type=QuantityOrSensor("MW"), + type=TimeSeriesOrQuantityOrSensor("MW"), required=False, default=None, help="Storage production power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1278,7 +1278,7 @@ def create_schedule(ctx): @click.option( "--storage-efficiency", "storage_efficiency", - type=QuantityOrSensor("%", default_src_unit="dimensionless"), + type=TimeSeriesOrQuantityOrSensor("%", default_src_unit="dimensionless"), required=False, default="100%", help="Storage efficiency (e.g. 95% or 0.95) to use for the schedule," @@ -1421,7 +1421,7 @@ def add_schedule_for_storage( # noqa C901 else: unit = "MW" - scheduling_kwargs[key][field_name] = QuantityOrSensor(unit)._serialize( + scheduling_kwargs[key][field_name] = TimeSeriesOrQuantityOrSensor(unit)._serialize( value, None, None ) diff --git a/flexmeasures/data/schemas/__init__.py b/flexmeasures/data/schemas/__init__.py index 1df7317b4..dbf4ff7b3 100644 --- a/flexmeasures/data/schemas/__init__.py +++ b/flexmeasures/data/schemas/__init__.py @@ -5,7 +5,7 @@ from .account import AccountIdField from .generic_assets import GenericAssetIdField as AssetIdField from .locations import LatitudeField, LongitudeField -from .sensors import SensorIdField, QuantityOrSensor +from .sensors import SensorIdField, TimeSeriesOrQuantityOrSensor from .sources import DataSourceIdField as SourceIdField from .times import ( AwareDateTimeField, diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index ea3bff476..8a689eb43 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1,6 +1,6 @@ from marshmallow import Schema, fields, validate -from flexmeasures.data.schemas.sensors import QuantityOrSensor, SensorIdField +from flexmeasures.data.schemas.sensors import TimeSeriesOrQuantityOrSensor, SensorIdField class FlexContextSchema(Schema): @@ -8,19 +8,19 @@ class FlexContextSchema(Schema): This schema lists fields that can be used to describe sensors in the optimised portfolio """ - ems_power_capacity_in_mw = QuantityOrSensor( + ems_power_capacity_in_mw = TimeSeriesOrQuantityOrSensor( "MW", required=False, data_key="site-power-capacity", validate=validate.Range(min=0), ) - ems_production_capacity_in_mw = QuantityOrSensor( + ems_production_capacity_in_mw = TimeSeriesOrQuantityOrSensor( "MW", required=False, data_key="site-production-capacity", validate=validate.Range(min=0), ) - ems_consumption_capacity_in_mw = QuantityOrSensor( + ems_consumption_capacity_in_mw = TimeSeriesOrQuantityOrSensor( "MW", required=False, data_key="site-consumption-capacity", diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index ab052b5e0..fedaf2d70 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -15,7 +15,7 @@ from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.schemas.units import QuantityField -from flexmeasures.data.schemas.sensors import QuantityOrSensor, TimeSeriesOrSensor +from flexmeasures.data.schemas.sensors import TimeSeriesOrQuantityOrSensor, TimeSeriesOrQuantityOrSensor from flexmeasures.utils.unit_utils import ur @@ -59,30 +59,30 @@ class StorageFlexModelSchema(Schema): soc_min = fields.Float(validate=validate.Range(min=0), data_key="soc-min") soc_max = fields.Float(data_key="soc-max") - power_capacity_in_mw = QuantityOrSensor( + power_capacity_in_mw = TimeSeriesOrQuantityOrSensor( "MW", required=False, data_key="power-capacity" ) - consumption_capacity = QuantityOrSensor( + consumption_capacity = TimeSeriesOrQuantityOrSensor( "MW", data_key="consumption-capacity", required=False ) - production_capacity = QuantityOrSensor( + production_capacity = TimeSeriesOrQuantityOrSensor( "MW", data_key="production-capacity", required=False ) # Timezone placeholder for the soc_maxima, soc_minima and soc_targets fields are overridden in __init__ - soc_maxima = TimeSeriesOrSensor( + soc_maxima = TimeSeriesOrQuantityOrSensor( to_unit="MWh", timezone="placeholder", data_key="soc-maxima" ) - soc_minima = TimeSeriesOrSensor( + soc_minima = TimeSeriesOrQuantityOrSensor( to_unit="MWh", timezone="placeholder", data_key="soc-minima", value_validator=validate.Range(min=0), ) - soc_targets = TimeSeriesOrSensor( + soc_targets = TimeSeriesOrQuantityOrSensor( to_unit="MWh", timezone="placeholder", data_key="soc-targets" ) @@ -96,10 +96,10 @@ class StorageFlexModelSchema(Schema): data_key="soc-unit", ) # todo: allow unit to be set per field, using QuantityField("%", validate=validate.Range(min=0, max=1)) - charging_efficiency = QuantityOrSensor( + charging_efficiency = TimeSeriesOrQuantityOrSensor( "%", data_key="charging-efficiency", required=False ) - discharging_efficiency = QuantityOrSensor( + discharging_efficiency = TimeSeriesOrQuantityOrSensor( "%", data_key="discharging-efficiency", required=False ) @@ -107,31 +107,31 @@ class StorageFlexModelSchema(Schema): data_key="roundtrip-efficiency", required=False ) - storage_efficiency = QuantityOrSensor( + storage_efficiency = TimeSeriesOrQuantityOrSensor( "%", default_src_unit="dimensionless", data_key="storage-efficiency" ) prefer_charging_sooner = fields.Bool(data_key="prefer-charging-sooner") - soc_gain = fields.List(QuantityOrSensor("MW"), data_key="soc-gain", required=False) + soc_gain = fields.List(TimeSeriesOrQuantityOrSensor("MW"), data_key="soc-gain", required=False) soc_usage = fields.List( - QuantityOrSensor("MW"), data_key="soc-usage", required=False + TimeSeriesOrQuantityOrSensor("MW"), data_key="soc-usage", required=False ) def __init__(self, start: datetime, sensor: Sensor, *args, **kwargs): """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" self.start = start self.sensor = sensor - self.soc_maxima = TimeSeriesOrSensor( + self.soc_maxima = TimeSeriesOrQuantityOrSensor( to_unit="MWh", timezone=sensor.timezone, data_key="soc-maxima" ) - self.soc_minima = TimeSeriesOrSensor( + self.soc_minima = TimeSeriesOrQuantityOrSensor( to_unit="MWh", timezone=sensor.timezone, data_key="soc-minima", value_validator=validate.Range(min=0), ) - self.soc_targets = TimeSeriesOrSensor( + self.soc_targets = TimeSeriesOrQuantityOrSensor( to_unit="MWh", timezone=sensor.timezone, data_key="soc-targets" ) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 8ee33b411..1fb257080 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -204,88 +204,7 @@ def _serialize(self, sensor: Sensor, attr, data, **kwargs) -> int: return sensor.id -class QuantityOrSensor(MarshmallowClickMixin, fields.Field): - def __init__( - self, to_unit: str, default_src_unit: str | None = None, *args, **kwargs - ): - """Field for validating, serializing and deserializing a Quantity or a Sensor. - - NB any validators passed are only applied to Quantities. - For example, validate=validate.Range(min=0) will raise a ValidationError in case of negative quantities, - but will let pass any sensor that has recorded negative values. - - :param to_unit: unit in which the sensor or quantity should be convertible to - :param default_src_unit: what unit to use in case of getting a numeric value - """ - - _validate = kwargs.pop("validate", None) - super().__init__(*args, **kwargs) - if _validate is not None: - # Insert validation into self.validators so that multiple errors can be stored. - validator = RepurposeValidatorToIgnoreSensors(_validate) - self.validators.insert(0, validator) - self.to_unit = ur.Quantity(to_unit) - self.default_src_unit = default_src_unit - - @with_appcontext_if_needed() - def _deserialize( - self, value: str | dict[str, int], attr, obj, **kwargs - ) -> ur.Quantity | Sensor: - if isinstance(value, dict): - if "sensor" not in value: - raise FMValidationError( - "Dictionary provided but `sensor` key not found." - ) - sensor = SensorIdField(unit=self.to_unit)._deserialize( - value["sensor"], None, None - ) - - return sensor - - elif isinstance(value, str): - try: - return ur.Quantity(value).to(self.to_unit) - except DimensionalityError as e: - raise FMValidationError( - f"Cannot convert value `{value}` to '{self.to_unit}'" - ) from e - elif self.default_src_unit is not None: - return self._deserialize( - f"{value} {self.default_src_unit}", attr, obj, **kwargs - ) - else: - raise FMValidationError( - f"Unsupported value type. `{type(value)}` was provided but only dict and str are supported." - ) - - def _serialize( - self, value: ur.Quantity | Sensor, attr, data, **kwargs - ) -> str | dict[str, int]: - if isinstance(value, ur.Quantity): - return str(value.to(self.to_unit)) - elif isinstance(value, Sensor): - return dict(sensor=value.id) - else: - raise FMValidationError( - "Serialized Quantity Or Sensor needs to be of type int, float or Sensor" - ) - - def convert(self, value, param, ctx, **kwargs): - # case that the click default is defined in numeric values - if not isinstance(value, str): - return super().convert(value, param, ctx, **kwargs) - - _value = re.match(r"sensor:(\d+)", value) - - if _value is not None: - _value = {"sensor": int(_value.groups()[0])} - else: - _value = value - - return super().convert(_value, param, ctx, **kwargs) - - -class TimeSeriesOrSensor(MarshmallowClickMixin, fields.Field): +class TimeSeriesOrQuantityOrSensor(MarshmallowClickMixin, fields.Field): def __init__( self, to_unit, @@ -306,7 +225,12 @@ def __init__( :param timezone: Only used in case a time series is specified and one of the *timed events* in the time series uses a nominal duration, such as "P1D". """ + _validate = kwargs.pop("validate", None) super().__init__(*args, **kwargs) + if _validate is not None: + # Insert validation into self.validators so that multiple errors can be stored. + validator = RepurposeValidatorToIgnoreSensors(_validate) + self.validators.insert(0, validator) self.timezone = timezone self.value_validator = value_validator self.to_unit = ur.Quantity(to_unit) diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 3891b7b17..3bafb45b2 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -1,6 +1,6 @@ import pytest from flexmeasures import Sensor -from flexmeasures.data.schemas.sensors import QuantityOrSensor +from flexmeasures.data.schemas.sensors import TimeSeriesOrQuantityOrSensor from flexmeasures.utils.unit_utils import ur from marshmallow import ValidationError @@ -27,7 +27,7 @@ def test_quantity_or_sensor_deserialize( setup_dummy_sensors, sensor_id, src_quantity, dst_unit, fails ): - schema = QuantityOrSensor(to_unit=dst_unit) + schema = TimeSeriesOrQuantityOrSensor(to_unit=dst_unit) try: if sensor_id is None: @@ -55,7 +55,7 @@ def test_quantity_or_sensor_conversion( setup_dummy_sensors, src_quantity, expected_magnitude ): - schema = QuantityOrSensor(to_unit="MW") + schema = TimeSeriesOrQuantityOrSensor(to_unit="MW") assert schema.deserialize(src_quantity).magnitude == expected_magnitude @@ -81,7 +81,7 @@ def test_quantity_or_sensor_field( setup_dummy_sensors, sensor_id, input_param, dst_unit, fails, db ): - field = QuantityOrSensor(to_unit=dst_unit) + field = TimeSeriesOrQuantityOrSensor(to_unit=dst_unit) try: if sensor_id is None: From ba8e1ccab8b708d5e50d2d1d47bd21a4e96241e8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 18:15:04 +0700 Subject: [PATCH 13/47] refactor: reduce number of blank lines Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 1fb257080..9639e0b80 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -246,11 +246,9 @@ def _deserialize( raise FMValidationError( "Dictionary provided but `sensor` key not found." ) - sensor = SensorIdField(unit=self.to_unit)._deserialize( value["sensor"], None, None ) - return sensor elif isinstance(value, list): @@ -261,7 +259,6 @@ def _deserialize( ) ) ) - return field._deserialize(value, None, None) elif isinstance(value, str): From 392a118dfd1ad33065e9e7aeba6c3d25dfefdca5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 22:07:08 +0700 Subject: [PATCH 14/47] fix: deprecate old classes Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 21 +++++++++++++++++++ .../data/schemas/tests/test_sensor.py | 8 +++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 9639e0b80..45e781214 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -1,4 +1,6 @@ from __future__ import annotations + +from flask import current_app from marshmallow import ( Schema, fields, @@ -321,3 +323,22 @@ def __call__(self, value): if not isinstance(value, Sensor): self.original_validator(value) return value + + +class QuantityOrSensor(TimeSeriesOrQuantityOrSensor): + def __init__(self, *args, **kwargs): + """Deprecated class. Use `TimeSeriesOrQuantityOrSensor` instead.""" + current_app.logger.warning( + f"Class `TimeSeriesOrSensor` is deprecated. Use `TimeSeriesOrQuantityOrSensor` instead." + ) + super().__init__(*args, **kwargs) + + +class TimeSeriesOrSensor(TimeSeriesOrQuantityOrSensor): + def __init__(self, *args, **kwargs): + """Deprecated class. Use `TimeSeriesOrQuantityOrSensor` instead.""" + current_app.logger.warning( + f"Class `TimeSeriesOrSensor` is deprecated. Use `TimeSeriesOrQuantityOrSensor` instead." + ) + super().__init__(*args, **kwargs) + diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 3bafb45b2..3891b7b17 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -1,6 +1,6 @@ import pytest from flexmeasures import Sensor -from flexmeasures.data.schemas.sensors import TimeSeriesOrQuantityOrSensor +from flexmeasures.data.schemas.sensors import QuantityOrSensor from flexmeasures.utils.unit_utils import ur from marshmallow import ValidationError @@ -27,7 +27,7 @@ def test_quantity_or_sensor_deserialize( setup_dummy_sensors, sensor_id, src_quantity, dst_unit, fails ): - schema = TimeSeriesOrQuantityOrSensor(to_unit=dst_unit) + schema = QuantityOrSensor(to_unit=dst_unit) try: if sensor_id is None: @@ -55,7 +55,7 @@ def test_quantity_or_sensor_conversion( setup_dummy_sensors, src_quantity, expected_magnitude ): - schema = TimeSeriesOrQuantityOrSensor(to_unit="MW") + schema = QuantityOrSensor(to_unit="MW") assert schema.deserialize(src_quantity).magnitude == expected_magnitude @@ -81,7 +81,7 @@ def test_quantity_or_sensor_field( setup_dummy_sensors, sensor_id, input_param, dst_unit, fails, db ): - field = TimeSeriesOrQuantityOrSensor(to_unit=dst_unit) + field = QuantityOrSensor(to_unit=dst_unit) try: if sensor_id is None: From 258520e4beb63aaa48bd2f8031efe86002dd10b9 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 22:10:20 +0700 Subject: [PATCH 15/47] fix: duplicate import Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index fedaf2d70..d88577a66 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -15,7 +15,7 @@ from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.schemas.units import QuantityField -from flexmeasures.data.schemas.sensors import TimeSeriesOrQuantityOrSensor, TimeSeriesOrQuantityOrSensor +from flexmeasures.data.schemas.sensors import TimeSeriesOrQuantityOrSensor from flexmeasures.utils.unit_utils import ur From 14757766d1988e216cd7b631cf0a608374e4a69f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 22:46:50 +0700 Subject: [PATCH 16/47] docs: document status quo Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 45e781214..9f40c129c 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -223,7 +223,10 @@ def __init__( but will let pass any sensor that has recorded negative values. :param to_unit: Unit in which the sensor or quantity should be convertible to. - :param default_src_unit: What unit to use in case of getting a numeric value. + - Time series are assumed to be passed without a unit, so they aren't checked for convertibility. + - Quantities will already be converted to the given unit. + - Sensors are checked for convertibility, but the original sensor is returned, so its values are not yet converted. + :param default_src_unit: What unit to use in case of getting a numeric value. Does not apply to time series or sensors. :param timezone: Only used in case a time series is specified and one of the *timed events* in the time series uses a nominal duration, such as "P1D". """ From 5cecd86d704c4addbbf1c03728939f90f1722b93 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 23:24:41 +0700 Subject: [PATCH 17/47] style: black Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 6 +++--- flexmeasures/data/schemas/scheduling/__init__.py | 5 ++++- flexmeasures/data/schemas/scheduling/storage.py | 4 +++- flexmeasures/data/schemas/sensors.py | 1 - 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 5f17591da..532dee37e 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -1421,9 +1421,9 @@ def add_schedule_for_storage( # noqa C901 else: unit = "MW" - scheduling_kwargs[key][field_name] = TimeSeriesOrQuantityOrSensor(unit)._serialize( - value, None, None - ) + scheduling_kwargs[key][field_name] = TimeSeriesOrQuantityOrSensor( + unit + )._serialize(value, None, None) if as_job: job = create_scheduling_job(asset_or_sensor=power_sensor, **scheduling_kwargs) diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 8a689eb43..40915ceeb 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1,6 +1,9 @@ from marshmallow import Schema, fields, validate -from flexmeasures.data.schemas.sensors import TimeSeriesOrQuantityOrSensor, SensorIdField +from flexmeasures.data.schemas.sensors import ( + TimeSeriesOrQuantityOrSensor, + SensorIdField, +) class FlexContextSchema(Schema): diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index d88577a66..c5066e489 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -112,7 +112,9 @@ class StorageFlexModelSchema(Schema): ) prefer_charging_sooner = fields.Bool(data_key="prefer-charging-sooner") - soc_gain = fields.List(TimeSeriesOrQuantityOrSensor("MW"), data_key="soc-gain", required=False) + soc_gain = fields.List( + TimeSeriesOrQuantityOrSensor("MW"), data_key="soc-gain", required=False + ) soc_usage = fields.List( TimeSeriesOrQuantityOrSensor("MW"), data_key="soc-usage", required=False ) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 9f40c129c..e4d9b2106 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -344,4 +344,3 @@ def __init__(self, *args, **kwargs): f"Class `TimeSeriesOrSensor` is deprecated. Use `TimeSeriesOrQuantityOrSensor` instead." ) super().__init__(*args, **kwargs) - From dbb91a823aadffd3444a4e7ec550f26835a2185d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 20 Jul 2024 23:30:56 +0700 Subject: [PATCH 18/47] style: flake8 Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index e4d9b2106..a7f1a161d 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -332,7 +332,7 @@ class QuantityOrSensor(TimeSeriesOrQuantityOrSensor): def __init__(self, *args, **kwargs): """Deprecated class. Use `TimeSeriesOrQuantityOrSensor` instead.""" current_app.logger.warning( - f"Class `TimeSeriesOrSensor` is deprecated. Use `TimeSeriesOrQuantityOrSensor` instead." + "Class `TimeSeriesOrSensor` is deprecated. Use `TimeSeriesOrQuantityOrSensor` instead." ) super().__init__(*args, **kwargs) @@ -341,6 +341,6 @@ class TimeSeriesOrSensor(TimeSeriesOrQuantityOrSensor): def __init__(self, *args, **kwargs): """Deprecated class. Use `TimeSeriesOrQuantityOrSensor` instead.""" current_app.logger.warning( - f"Class `TimeSeriesOrSensor` is deprecated. Use `TimeSeriesOrQuantityOrSensor` instead." + "Class `TimeSeriesOrSensor` is deprecated. Use `TimeSeriesOrQuantityOrSensor` instead." ) super().__init__(*args, **kwargs) From 9478c5a7495af2beaa1c91397aa5d42107b61432 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 23 Jul 2024 22:39:32 +0700 Subject: [PATCH 19/47] feature: allow interpreting float values as quantities based on a unit defined elsewhere, and specifically, let the StorageScheduler get its default SoC unit from the soc-unit field, which lets us allow time series values to be specified as string quantities while preserving backwards compatibility Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- .../data/schemas/scheduling/storage.py | 15 ++++++-- flexmeasures/data/schemas/sensors.py | 16 +++++++- .../data/schemas/tests/test_sensor.py | 37 ++++++++++++++++--- flexmeasures/data/schemas/units.py | 12 +++++- 5 files changed, 68 insertions(+), 14 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index a30bec654..c825c39bb 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -475,7 +475,7 @@ def deserialize_flex_config(self): # Now it's time to check if our flex configurations holds up to schemas self.flex_model = StorageFlexModelSchema( - start=self.start, sensor=self.sensor + start=self.start, sensor=self.sensor, default_soc_unit=self.flex_model["soc-unit"] ).load(self.flex_model) self.flex_context = FlexContextSchema().load(self.flex_context) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index c5066e489..a38f19a36 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -70,7 +70,7 @@ class StorageFlexModelSchema(Schema): "MW", data_key="production-capacity", required=False ) - # Timezone placeholder for the soc_maxima, soc_minima and soc_targets fields are overridden in __init__ + # Timezone placeholders for the soc_maxima, soc_minima and soc_targets fields are overridden in __init__ soc_maxima = TimeSeriesOrQuantityOrSensor( to_unit="MWh", timezone="placeholder", data_key="soc-maxima" ) @@ -119,22 +119,29 @@ class StorageFlexModelSchema(Schema): TimeSeriesOrQuantityOrSensor("MW"), data_key="soc-usage", required=False ) - def __init__(self, start: datetime, sensor: Sensor, *args, **kwargs): + def __init__(self, start: datetime, sensor: Sensor, *args, default_soc_unit: str | None = None, **kwargs): """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" self.start = start self.sensor = sensor self.soc_maxima = TimeSeriesOrQuantityOrSensor( - to_unit="MWh", timezone=sensor.timezone, data_key="soc-maxima" + to_unit="MWh", + default_src_unit=default_soc_unit, + timezone=sensor.timezone, + data_key="soc-maxima", ) self.soc_minima = TimeSeriesOrQuantityOrSensor( to_unit="MWh", + default_src_unit=default_soc_unit, timezone=sensor.timezone, data_key="soc-minima", value_validator=validate.Range(min=0), ) self.soc_targets = TimeSeriesOrQuantityOrSensor( - to_unit="MWh", timezone=sensor.timezone, data_key="soc-targets" + to_unit="MWh", + default_src_unit=default_soc_unit, + timezone=sensor.timezone, + data_key="soc-targets", ) super().__init__(*args, **kwargs) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index a7f1a161d..cb1b5e95c 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -27,6 +27,7 @@ ) from flexmeasures.utils.unit_utils import is_valid_unit, ur, units_are_convertible from flexmeasures.data.schemas.times import DurationField, AwareDateTimeField +from flexmeasures.data.schemas.units import QuantityField class JSON(fields.Field): @@ -41,7 +42,11 @@ def _serialize(self, value, attr, data, **kwargs) -> str: class TimedEventSchema(Schema): - value = fields.Float(required=True) + value = QuantityField( + required=True, + to_unit="dimensionless", # placeholder, overridden in __init__ + default_src_unit="dimensionless", # placeholder, overridden in __init__ + ) datetime = AwareDateTimeField(required=False) start = AwareDateTimeField(required=False) end = AwareDateTimeField(required=False) @@ -51,6 +56,8 @@ def __init__( self, timezone: str | None = None, value_validator: Validator | None = None, + to_unit: str | None = None, + default_src_unit: str | None = None, *args, **kwargs, ): @@ -61,6 +68,8 @@ def __init__( self.timezone = timezone self.value_validator = value_validator super().__init__(*args, **kwargs) + setattr(self.fields["value"], "to_unit", to_unit) + setattr(self.fields["value"], "default_src_unit", default_src_unit) @validates("value") def validate_value(self, _value): @@ -260,7 +269,10 @@ def _deserialize( field = fields.List( fields.Nested( TimedEventSchema( - timezone=self.timezone, value_validator=self.value_validator + timezone=self.timezone, + value_validator=self.value_validator, + to_unit=self.to_unit, + default_src_unit=self.default_src_unit, ) ) ) diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 3891b7b17..a32132742 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -1,6 +1,6 @@ import pytest from flexmeasures import Sensor -from flexmeasures.data.schemas.sensors import QuantityOrSensor +from flexmeasures.data.schemas.sensors import QuantityOrSensor, TimeSeriesOrQuantityOrSensor from flexmeasures.utils.unit_utils import ur from marshmallow import ValidationError @@ -35,8 +35,8 @@ def test_quantity_or_sensor_deserialize( else: schema.deserialize({"sensor": sensor_id}) assert not fails - except ValidationError: - assert fails + except ValidationError as e: + assert fails, e @pytest.mark.parametrize( @@ -92,5 +92,32 @@ def test_quantity_or_sensor_field( assert val == db.session.get(Sensor, sensor_id) assert not fails - except Exception: - assert fails + except Exception as e: + assert fails, e + + +@pytest.mark.parametrize( + "input_param, dst_unit, fails", + [ + # deserialize a quantity + ([{"value": 1, "datetime": "2024-07-21T00:15+07"}], "MWh", False), + ([{"value": "1", "datetime": "2024-07-21T00:15+07"}], "MWh", True), + ([{"value": "1MWh", "datetime": "2024-07-21T00:15+07"}], "MWh", False), + ([{"value": "1000 kWh", "datetime": "2024-07-21T00:15+07"}], "MWh", False), + ([{"value": "1 MW", "datetime": "2024-07-21T00:15+07"}], "MWh", True), + ], +) +def test_time_series_field( + input_param, dst_unit, fails, db +): + + field = TimeSeriesOrQuantityOrSensor(to_unit=dst_unit, default_src_unit="MWh") + + try: + val = field.convert(input_param, None, None) + assert val[0]["value"].units == ur.Unit(dst_unit) + assert val[0]["value"].magnitude == 1 + + assert not fails + except Exception as e: + assert fails, e diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py index 99803ca45..4b3ba6c49 100644 --- a/flexmeasures/data/schemas/units.py +++ b/flexmeasures/data/schemas/units.py @@ -34,16 +34,24 @@ class QuantityField(MarshmallowClickMixin, fields.Str): """ - def __init__(self, to_unit: str, *args, **kwargs): + def __init__(self, to_unit: str, *args, default_src_unit: str | None = None, **kwargs): super().__init__(*args, **kwargs) # Insert validation into self.validators so that multiple errors can be stored. validator = QuantityValidator() self.validators.insert(0, validator) self.to_unit = ur.Quantity(to_unit) + self.default_src_unit = default_src_unit def _deserialize(self, value, attr, obj, **kwargs) -> ur.Quantity: """Turn a quantity describing string into a Quantity.""" - return ur.Quantity(value).to(self.to_unit) + if isinstance(value, str): + return ur.Quantity(value).to(self.to_unit) + elif self.default_src_unit is not None: + return self._deserialize( + f"{value} {self.default_src_unit}", attr, obj, **kwargs + ) + else: + self._deserialize(f"{value}", attr, obj, **kwargs) def _serialize(self, value, attr, data, **kwargs): """Turn a Quantity into a string in scientific format.""" From ba26a2e1207680b71ad218563dcc4ba2f8014d61 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Jul 2024 00:14:52 +0700 Subject: [PATCH 20/47] fix: maintain backwards compatibility for transforming Float fields into Quantity fields, by returning the magnitude upon deserialization Signed-off-by: F.N. Claessen --- .../data/models/planning/tests/test_solver.py | 3 ++ .../data/schemas/scheduling/storage.py | 43 ++++++++++++++++--- flexmeasures/data/schemas/sensors.py | 19 +++++++- .../data/schemas/tests/test_scheduling.py | 2 +- .../data/schemas/tests/test_sensor.py | 6 ++- flexmeasures/data/schemas/units.py | 32 +++++++++++--- 6 files changed, 89 insertions(+), 16 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index e436e16a4..3c7f1a81a 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -674,6 +674,7 @@ def test_soc_bounds_timeseries(db, add_battery_assets): resolution = timedelta(hours=1) # soc parameters + soc_unit = "MWh" soc_at_start = battery.get_attribute("soc_in_mwh") soc_min = 0.5 soc_max = 4.5 @@ -697,6 +698,7 @@ def compute_schedule(flex_model): return soc_schedule flex_model = { + "soc-unit": soc_unit, "soc-at-start": soc_at_start, "soc-min": soc_min, "soc-max": soc_max, @@ -715,6 +717,7 @@ def compute_schedule(flex_model): soc_targets = [{"datetime": "2015-01-02T19:00:00+01:00", "value": 2.0}] flex_model = { + "soc-unit": soc_unit, "soc-at-start": soc_at_start, "soc-min": soc_min, "soc-max": soc_max, diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index a38f19a36..b44f35cb8 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -54,10 +54,27 @@ class StorageFlexModelSchema(Schema): You can use StorageScheduler.deserialize_flex_config to get that filled in. """ - soc_at_start = fields.Float(required=True, data_key="soc-at-start") + soc_at_start = QuantityField( + required=True, + to_unit="MWh", + default_src_unit="dimensionless", # placeholder, overridden in __init__ + return_magnitude=True, + data_key="soc-at-start", + ) - soc_min = fields.Float(validate=validate.Range(min=0), data_key="soc-min") - soc_max = fields.Float(data_key="soc-max") + soc_min = QuantityField( + validate=validate.Range(min=0), + to_unit="MWh", + default_src_unit="dimensionless", # placeholder, overridden in __init__ + return_magnitude=True, + data_key="soc-min", + ) + soc_max = QuantityField( + to_unit="MWh", + default_src_unit="dimensionless", # placeholder, overridden in __init__ + return_magnitude=True, + data_key="soc-max", + ) power_capacity_in_mw = TimeSeriesOrQuantityOrSensor( "MW", required=False, data_key="power-capacity" @@ -72,18 +89,25 @@ class StorageFlexModelSchema(Schema): # Timezone placeholders for the soc_maxima, soc_minima and soc_targets fields are overridden in __init__ soc_maxima = TimeSeriesOrQuantityOrSensor( - to_unit="MWh", timezone="placeholder", data_key="soc-maxima" + to_unit="MWh", + default_src_unit="dimensionless", # placeholder, overridden in __init__ + timezone="placeholder", + data_key="soc-maxima", ) soc_minima = TimeSeriesOrQuantityOrSensor( to_unit="MWh", + default_src_unit="dimensionless", # placeholder, overridden in __init__ timezone="placeholder", data_key="soc-minima", value_validator=validate.Range(min=0), ) soc_targets = TimeSeriesOrQuantityOrSensor( - to_unit="MWh", timezone="placeholder", data_key="soc-targets" + to_unit="MWh", + default_src_unit="dimensionless", # placeholder, overridden in __init__ + timezone="placeholder", + data_key="soc-targets", ) soc_unit = fields.Str( @@ -94,7 +118,7 @@ class StorageFlexModelSchema(Schema): ] ), data_key="soc-unit", - ) # todo: allow unit to be set per field, using QuantityField("%", validate=validate.Range(min=0, max=1)) + ) charging_efficiency = TimeSeriesOrQuantityOrSensor( "%", data_key="charging-efficiency", required=False @@ -145,6 +169,13 @@ def __init__(self, start: datetime, sensor: Sensor, *args, default_soc_unit: str ) super().__init__(*args, **kwargs) + if default_soc_unit is not None: + setattr(self.fields["soc_at_start"], "default_src_unit", default_soc_unit) + setattr(self.fields["soc_min"], "default_src_unit", default_soc_unit) + setattr(self.fields["soc_max"], "default_src_unit", default_soc_unit) + setattr(self.fields["soc_minima"], "default_src_unit", default_soc_unit) + setattr(self.fields["soc_maxima"], "default_src_unit", default_soc_unit) + setattr(self.fields["soc_targets"], "default_src_unit", default_soc_unit) @validates_schema def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs): diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index cb1b5e95c..c9ca95291 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -46,6 +46,7 @@ class TimedEventSchema(Schema): required=True, to_unit="dimensionless", # placeholder, overridden in __init__ default_src_unit="dimensionless", # placeholder, overridden in __init__ + return_magnitude=True, # placeholder, overridden in __init__ ) datetime = AwareDateTimeField(required=False) start = AwareDateTimeField(required=False) @@ -58,6 +59,7 @@ def __init__( value_validator: Validator | None = None, to_unit: str | None = None, default_src_unit: str | None = None, + return_magnitude: bool = True, *args, **kwargs, ): @@ -68,8 +70,11 @@ def __init__( self.timezone = timezone self.value_validator = value_validator super().__init__(*args, **kwargs) - setattr(self.fields["value"], "to_unit", to_unit) - setattr(self.fields["value"], "default_src_unit", default_src_unit) + if to_unit is not None: + setattr(self.fields["value"], "to_unit", to_unit) + if default_src_unit is not None: + setattr(self.fields["value"], "default_src_unit", default_src_unit) + setattr(self.fields["value"], "return_magnitude", return_magnitude) @validates("value") def validate_value(self, _value): @@ -221,12 +226,15 @@ def __init__( to_unit, *args, default_src_unit: str | None = None, + return_magnitude: bool = True, timezone: str | None = None, value_validator: Validator | None = None, **kwargs, ): """Field for validating, serializing and deserializing a quantity, sensor or time series. + # todo: Sensor should perhaps deserialize already to sensor data + NB any validators passed are only applied to Quantities. For example, validate=validate.Range(min=0) will raise a ValidationError in case of negative quantities, but will let pass any sensor that has recorded negative values. @@ -236,6 +244,7 @@ def __init__( - Quantities will already be converted to the given unit. - Sensors are checked for convertibility, but the original sensor is returned, so its values are not yet converted. :param default_src_unit: What unit to use in case of getting a numeric value. Does not apply to time series or sensors. + :param return_magnitude: In case of getting a time series, whether the result should include the magnitude of each quantity, or each Quantity object itself :param timezone: Only used in case a time series is specified and one of the *timed events* in the time series uses a nominal duration, such as "P1D". """ @@ -249,6 +258,7 @@ def __init__( self.value_validator = value_validator self.to_unit = ur.Quantity(to_unit) self.default_src_unit = default_src_unit + self.return_magnitude = return_magnitude @with_appcontext_if_needed() def _deserialize( @@ -266,6 +276,10 @@ def _deserialize( return sensor elif isinstance(value, list): + if self.return_magnitude is True: + current_app.logger.warning( + "Deserialized time series will include Quantity objects in the future. Set `return_magnitude=False` to trigger the new behaviour." + ) field = fields.List( fields.Nested( TimedEventSchema( @@ -273,6 +287,7 @@ def _deserialize( value_validator=self.value_validator, to_unit=self.to_unit, default_src_unit=self.default_src_unit, + return_magnitude=self.return_magnitude, ) ) ) diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 2d18e85ef..27e554c1a 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -204,7 +204,7 @@ def test_efficiency_pair( def load_schema(): flex_model = { "storage-efficiency": 1, - "soc-at-start": 0, + "soc-at-start": "0 MWh", } for f in fields: flex_model[f] = "90%" diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index a32132742..d9615b8ce 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -111,7 +111,11 @@ def test_time_series_field( input_param, dst_unit, fails, db ): - field = TimeSeriesOrQuantityOrSensor(to_unit=dst_unit, default_src_unit="MWh") + field = TimeSeriesOrQuantityOrSensor( + to_unit=dst_unit, + default_src_unit="MWh", + return_magnitude=False, + ) try: val = field.convert(input_param, None, None) diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py index 4b3ba6c49..323e43f64 100644 --- a/flexmeasures/data/schemas/units.py +++ b/flexmeasures/data/schemas/units.py @@ -34,24 +34,44 @@ class QuantityField(MarshmallowClickMixin, fields.Str): """ - def __init__(self, to_unit: str, *args, default_src_unit: str | None = None, **kwargs): + def __init__( + self, + to_unit: str, + *args, + default_src_unit: str | None = None, + return_magnitude: bool = False, + **kwargs, + ): super().__init__(*args, **kwargs) # Insert validation into self.validators so that multiple errors can be stored. validator = QuantityValidator() self.validators.insert(0, validator) self.to_unit = ur.Quantity(to_unit) self.default_src_unit = default_src_unit + self.return_magnitude = return_magnitude - def _deserialize(self, value, attr, obj, **kwargs) -> ur.Quantity: + def _deserialize( + self, + value, + attr, + obj, + return_magnitude: bool | None = None, + **kwargs, + ) -> ur.Quantity: """Turn a quantity describing string into a Quantity.""" + if return_magnitude is None: + return_magnitude = self.return_magnitude if isinstance(value, str): - return ur.Quantity(value).to(self.to_unit) + q = ur.Quantity(value).to(self.to_unit) elif self.default_src_unit is not None: - return self._deserialize( - f"{value} {self.default_src_unit}", attr, obj, **kwargs + q = self._deserialize( + f"{value} {self.default_src_unit}", attr, obj, **kwargs, return_magnitude=False ) else: - self._deserialize(f"{value}", attr, obj, **kwargs) + q = self._deserialize(f"{value}", attr, obj, **kwargs, return_magnitude=False) + if return_magnitude: + return q.magnitude + return q def _serialize(self, value, attr, data, **kwargs): """Turn a Quantity into a string in scientific format.""" From ead17b80e4a0987fe9a293a405a071e0e5c4b242 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Jul 2024 09:16:23 +0700 Subject: [PATCH 21/47] docs: grammar Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index c825c39bb..fef414448 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -473,7 +473,7 @@ def deserialize_flex_config(self): self.ensure_soc_min_max() - # Now it's time to check if our flex configurations holds up to schemas + # Now it's time to check if our flex configuration holds up to schemas self.flex_model = StorageFlexModelSchema( start=self.start, sensor=self.sensor, default_soc_unit=self.flex_model["soc-unit"] ).load(self.flex_model) From c847870e20cc9fd89a4d4b5344decf1cc169cd1c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Jul 2024 09:18:46 +0700 Subject: [PATCH 22/47] docs: add todo Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index fef414448..ba56b1a53 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -470,6 +470,9 @@ def deserialize_flex_config(self): self.flex_model["soc-unit"] = self.sensor.unit elif self.sensor.unit in ("MW", "kW"): self.flex_model["soc-unit"] = self.sensor.unit + "h" + else: + # todo: raise? Surely we must have a unit. + pass self.ensure_soc_min_max() From 137f6ae9bdcc766751583c32240d92c86e63906a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Jul 2024 22:24:08 +0700 Subject: [PATCH 23/47] fix: tests Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_sensor_schedules.py | 2 +- flexmeasures/data/models/planning/storage.py | 12 +------ .../data/schemas/scheduling/storage.py | 31 ++----------------- flexmeasures/data/schemas/units.py | 2 ++ 4 files changed, 7 insertions(+), 40 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index be7b67760..3965cf5e7 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -46,7 +46,7 @@ def test_get_schedule_wrong_job_id( message_for_trigger_schedule(), "soc-min", "not-a-float", - "Not a valid number", + "Not a valid quantity", ), (message_for_trigger_schedule(), "soc-unit", "MWH", "Must be one of"), # todo: add back test in case we stop grandfathering ignoring too-far-into-the-future targets, or amend otherwise diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index ba56b1a53..a033ae011 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -420,17 +420,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 def persist_flex_model(self): """Store new soc info as GenericAsset attributes""" self.sensor.generic_asset.set_attribute("soc_datetime", self.start.isoformat()) - soc_unit = self.flex_model.get("soc_unit") - if soc_unit == "kWh": - self.sensor.generic_asset.set_attribute( - "soc_in_mwh", self.flex_model["soc_at_start"] / 1000 - ) - elif soc_unit == "MWh": - self.sensor.generic_asset.set_attribute( - "soc_in_mwh", self.flex_model["soc_at_start"] - ) - else: - raise NotImplementedError(f"Unsupported SoC unit '{soc_unit}'.") + self.sensor.generic_asset.set_attribute("soc_in_mwh", self.flex_model["soc_at_start"]) def deserialize_flex_config(self): """ diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index b44f35cb8..b36b2dfc4 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -63,7 +63,7 @@ class StorageFlexModelSchema(Schema): ) soc_min = QuantityField( - validate=validate.Range(min=0), + validate=validate.Range(min=0), # change to min=ur.Quantity("0 MWh") in case return_magnitude=False to_unit="MWh", default_src_unit="dimensionless", # placeholder, overridden in __init__ return_magnitude=True, @@ -226,33 +226,8 @@ def check_redundant_efficiencies(self, data: dict, **kwargs): @post_load def post_load_sequence(self, data: dict, **kwargs) -> dict: """Perform some checks and corrections after we loaded.""" - # currently we only handle MWh internally - # TODO: review when we moved away from capacity having to be described in MWh - if data.get("soc_unit") == "kWh": - data["soc_at_start"] /= 1000.0 - if data.get("soc_min") is not None: - data["soc_min"] /= 1000.0 - if data.get("soc_max") is not None: - data["soc_max"] /= 1000.0 - if ( - not isinstance(data.get("soc_targets"), Sensor) - and data.get("soc_targets") is not None - ): - for target in data["soc_targets"]: - target["value"] /= 1000.0 - if ( - not isinstance(data.get("soc_minima"), Sensor) - and data.get("soc_minima") is not None - ): - for minimum in data["soc_minima"]: - minimum["value"] /= 1000.0 - if ( - not isinstance(data.get("soc_maxima"), Sensor) - and data.get("soc_maxima") is not None - ): - for maximum in data["soc_maxima"]: - maximum["value"] /= 1000.0 - data["soc_unit"] = "MWh" + # currently we only handle MWh internally, and the conversion to MWh happened during deserialization + data["soc_unit"] = "MWh" # Convert efficiency to dimensionless (to the (0,1] range) if data.get("roundtrip_efficiency") is not None: diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py index 323e43f64..e266a786b 100644 --- a/flexmeasures/data/schemas/units.py +++ b/flexmeasures/data/schemas/units.py @@ -62,6 +62,8 @@ def _deserialize( if return_magnitude is None: return_magnitude = self.return_magnitude if isinstance(value, str): + if not is_valid_unit(value): + raise ValidationError("Not a valid quantity") q = ur.Quantity(value).to(self.to_unit) elif self.default_src_unit is not None: q = self._deserialize( From 00bf425b9cc5b4c9463081abc9a05f9a5f15e34c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Jul 2024 23:00:40 +0700 Subject: [PATCH 24/47] style: black Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 8 ++++++-- flexmeasures/data/schemas/scheduling/storage.py | 13 +++++++++++-- flexmeasures/data/schemas/tests/test_sensor.py | 9 +++++---- flexmeasures/data/schemas/units.py | 10 ++++++++-- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index a033ae011..1bd279c51 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -420,7 +420,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 def persist_flex_model(self): """Store new soc info as GenericAsset attributes""" self.sensor.generic_asset.set_attribute("soc_datetime", self.start.isoformat()) - self.sensor.generic_asset.set_attribute("soc_in_mwh", self.flex_model["soc_at_start"]) + self.sensor.generic_asset.set_attribute( + "soc_in_mwh", self.flex_model["soc_at_start"] + ) def deserialize_flex_config(self): """ @@ -468,7 +470,9 @@ def deserialize_flex_config(self): # Now it's time to check if our flex configuration holds up to schemas self.flex_model = StorageFlexModelSchema( - start=self.start, sensor=self.sensor, default_soc_unit=self.flex_model["soc-unit"] + start=self.start, + sensor=self.sensor, + default_soc_unit=self.flex_model["soc-unit"], ).load(self.flex_model) self.flex_context = FlexContextSchema().load(self.flex_context) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index b36b2dfc4..8d41d6bf1 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -63,7 +63,9 @@ class StorageFlexModelSchema(Schema): ) soc_min = QuantityField( - validate=validate.Range(min=0), # change to min=ur.Quantity("0 MWh") in case return_magnitude=False + validate=validate.Range( + min=0 + ), # change to min=ur.Quantity("0 MWh") in case return_magnitude=False to_unit="MWh", default_src_unit="dimensionless", # placeholder, overridden in __init__ return_magnitude=True, @@ -143,7 +145,14 @@ class StorageFlexModelSchema(Schema): TimeSeriesOrQuantityOrSensor("MW"), data_key="soc-usage", required=False ) - def __init__(self, start: datetime, sensor: Sensor, *args, default_soc_unit: str | None = None, **kwargs): + def __init__( + self, + start: datetime, + sensor: Sensor, + *args, + default_soc_unit: str | None = None, + **kwargs, + ): """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" self.start = start self.sensor = sensor diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index d9615b8ce..ff7335c97 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -1,6 +1,9 @@ import pytest from flexmeasures import Sensor -from flexmeasures.data.schemas.sensors import QuantityOrSensor, TimeSeriesOrQuantityOrSensor +from flexmeasures.data.schemas.sensors import ( + QuantityOrSensor, + TimeSeriesOrQuantityOrSensor, +) from flexmeasures.utils.unit_utils import ur from marshmallow import ValidationError @@ -107,9 +110,7 @@ def test_quantity_or_sensor_field( ([{"value": "1 MW", "datetime": "2024-07-21T00:15+07"}], "MWh", True), ], ) -def test_time_series_field( - input_param, dst_unit, fails, db -): +def test_time_series_field(input_param, dst_unit, fails, db): field = TimeSeriesOrQuantityOrSensor( to_unit=dst_unit, diff --git a/flexmeasures/data/schemas/units.py b/flexmeasures/data/schemas/units.py index e266a786b..47ef861cc 100644 --- a/flexmeasures/data/schemas/units.py +++ b/flexmeasures/data/schemas/units.py @@ -67,10 +67,16 @@ def _deserialize( q = ur.Quantity(value).to(self.to_unit) elif self.default_src_unit is not None: q = self._deserialize( - f"{value} {self.default_src_unit}", attr, obj, **kwargs, return_magnitude=False + f"{value} {self.default_src_unit}", + attr, + obj, + **kwargs, + return_magnitude=False, ) else: - q = self._deserialize(f"{value}", attr, obj, **kwargs, return_magnitude=False) + q = self._deserialize( + f"{value}", attr, obj, **kwargs, return_magnitude=False + ) if return_magnitude: return q.magnitude return q From cc68ccf91c0f812ce79edaf8b75d76b2584ed8fd Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Jul 2024 23:44:30 +0700 Subject: [PATCH 25/47] docs: update V2G flex-model Signed-off-by: F.N. Claessen --- documentation/tut/flex-model-v2g.rst | 42 ++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/documentation/tut/flex-model-v2g.rst b/documentation/tut/flex-model-v2g.rst index 3d44e703b..4d92af527 100644 --- a/documentation/tut/flex-model-v2g.rst +++ b/documentation/tut/flex-model-v2g.rst @@ -33,9 +33,8 @@ Constraining the cycling to occur within a static 25-85% SoC range can be modell { "flex-model": { - "soc-min": 15, - "soc-max": 51, - "soc-unit": "kWh" + "soc-min": "15 kWh", + "soc-max": "51 kWh" } } @@ -50,16 +49,15 @@ To enable a temporary target SoC of more than 85% (for car reservations, see the { "flex-model": { - "soc-min": 15, - "soc-max": 60, + "soc-min": "15 kWh", + "soc-max": "60 kWh", "soc-maxima": [ { - "value": 51, + "value": "51 kWh", "start": "2024-02-04T10:35:00+01:00", "end": "2024-02-05T04:25:00+01:00" } - ], - "soc-unit": "kWh" + ] } } @@ -80,7 +78,7 @@ Given a reservation for 8 AM on February 5th, constraint 2 can be modelled throu "flex-model": { "soc-minima": [ { - "value": 57, + "value": "57 kWh", "datetime": "2024-02-05T08:00:00+01:00" } ] @@ -88,7 +86,7 @@ Given a reservation for 8 AM on February 5th, constraint 2 can be modelled throu } This constraint also signals that if the car is not plugged out of the Charge Point at 8 AM, the scheduler is in principle allowed to start discharging immediately afterwards. -To make sure the car remains at 95% SoC for some time, additional soc-minima constraints should be set accordingly, taking into account the scheduling resolution (here, 5 minutes). For example, to keep it charged (nearly) fully until 8.15 AM: +To make sure the car remains at or above 95% SoC for some time, additional soc-minima constraints should be set accordingly, taking into account the scheduling resolution (here, 5 minutes). For example, to keep it charged (nearly) fully until 8.15 AM: .. code-block:: json @@ -96,7 +94,7 @@ To make sure the car remains at 95% SoC for some time, additional soc-minima con "flex-model": { "soc-minima": [ { - "value": 57, + "value": "57 kWh", "start": "2024-02-05T08:00:00+01:00", "end": "2024-02-05T08:15:00+01:00" } @@ -104,6 +102,28 @@ To make sure the car remains at 95% SoC for some time, additional soc-minima con } } +The car may still charge and discharge within those 15 minutes, but it won't go below 95%. +Alternatively, to keep the car from discharging altogether during that time, limit the ``production-capacity`` (likewise, use the ``consumption-capacity`` to prevent any charging): + +.. code-block:: json + + { + "flex-model": { + "soc-minima": [ + { + "value": "57 kWh", + "datetime": "2024-02-05T08:00:00+01:00" + } + ], + "production-capacity": [ + { + "value": "0 kW", + "start": "2024-02-05T08:00:00+01:00", + "end": "2024-02-05T08:15:00+01:00" + } + ] + } + } .. _earning_by_cycling: From b29dad24f13406eac69033f705835013701e2342 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Jul 2024 23:48:02 +0700 Subject: [PATCH 26/47] docs: update trigger endpoint examples Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/sensors.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 63b912c20..6d04dd7a7 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -279,8 +279,7 @@ def trigger_schedule( { "start": "2015-06-02T10:00:00+00:00", "flex-model": { - "soc-at-start": 12.1, - "soc-unit": "kWh" + "soc-at-start": "12.1 kWh" } } @@ -311,17 +310,16 @@ def trigger_schedule( "start": "2015-06-02T10:00:00+00:00", "duration": "PT24H", "flex-model": { - "soc-at-start": 12.1, - "soc-unit": "kWh", + "soc-at-start": "12.1 kWh", "soc-targets": [ { - "value": 25, + "value": "25 kWh", "datetime": "2015-06-02T16:00:00+00:00" }, ], "soc-minima": {"sensor" : 300}, - "soc-min": 10, - "soc-max": 25, + "soc-min": "10 kWh", + "soc-max": "25 kWh", "charging-efficiency": "120%", "discharging-efficiency": {"sensor": 98}, "storage-efficiency": 0.9999, From ee10c1934c5a487303178e4c79e78f6dfea60d38 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Wed, 24 Jul 2024 23:50:39 +0700 Subject: [PATCH 27/47] docs: update other tutorials Signed-off-by: F.N. Claessen --- documentation/tut/forecasting_scheduling.rst | 7 +++---- documentation/tut/posting_data.rst | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/documentation/tut/forecasting_scheduling.rst b/documentation/tut/forecasting_scheduling.rst index 15152e90b..1d259a47e 100644 --- a/documentation/tut/forecasting_scheduling.rst +++ b/documentation/tut/forecasting_scheduling.rst @@ -108,18 +108,17 @@ Here, we extend that (storage) example with an additional target value, represen { "start": "2015-06-02T10:00:00+00:00", "flex-model": { - "soc-at-start": 12.1, - "soc-unit": "kWh" + "soc-at-start": "12.1 kWh", "soc-targets": [ { - "value": 25, + "value": "25 kWh", "datetime": "2015-06-02T16:00:00+00:00" } } } -We now have described the state of charge at 10am to be ``12.1``. In addition, we requested that it should be ``25`` at 4pm. +We now have described the state of charge at 10am to be ``"12.1 kWh"``. In addition, we requested that it should be ``"25 kWh"`` at 4pm. For instance, this could mean that a car should be charged at 90% at that time. If FlexMeasures receives this message, a scheduling job will be made and put into the queue. In turn, the scheduling job creates a proposed schedule. We'll look a bit deeper into those further down in :ref:`getting_schedules`. diff --git a/documentation/tut/posting_data.rst b/documentation/tut/posting_data.rst index ac60bafea..4189c136e 100644 --- a/documentation/tut/posting_data.rst +++ b/documentation/tut/posting_data.rst @@ -287,8 +287,7 @@ The endpoint also allows to limit the flexibility range and also to set target v { "start": "2015-06-02T10:00:00+00:00", "flex-model": { - "soc-at-start": 12.1, - "soc-unit": "kWh" + "soc-at-start": "12.1 kWh" } } From fe4d755952d315146d4b606a4f174f9c24630b30 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 25 Jul 2024 00:02:12 +0700 Subject: [PATCH 28/47] docs: update scheduling feature section Signed-off-by: F.N. Claessen --- documentation/features/scheduling.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 6d5410b13..13fabb66f 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -98,25 +98,25 @@ and what constraints or preferences should be taken into account. - Example value - Description * - ``soc-at-start`` - - ``"3.1"`` + - ``"3.1 kWh"`` - The (estimated) state of charge at the beginning of the schedule (defaults to 0). * - ``soc-unit`` - ``"kWh"`` or ``"MWh"`` - - The unit in which all SoC related flex-model values are to be interpreted. + - The unit used to interpret any SoC related flex-model value that does not mention a unit itself (only applies to numeric values, so not to string values). * - ``soc-min`` - - ``"2.5"`` + - ``"2.5 kWh"`` - A constant lower boundary for all values in the schedule (defaults to 0). * - ``soc-max`` - - ``"7"`` + - ``"7 kWh"`` - A constant upper boundary for all values in the schedule (defaults to max soc target, if provided) * - ``soc-minima`` - - ``[{"datetime": "2024-02-05T08:00:00+01:00", value: 8.2}]`` + - ``[{"datetime": "2024-02-05T08:00:00+01:00", value: "8.2 kWh"}]`` - Set point(s) that form lower boundaries, e.g. to target a full car battery in the morning. Can be single values or a range (defaults to NaN values). * - ``soc-maxima`` - - ``{"value": 51, "start": "2024-02-05T12:00:00+01:00","end": "2024-02-05T13:30:00+01:00"}`` + - ``{"value": "51 kWh", "start": "2024-02-05T12:00:00+01:00", "end": "2024-02-05T13:30:00+01:00"}`` - Set point(s) that form upper boundaries at certain times. Can be single values or a range (defaults to NaN values). * - ``soc-targets`` - - ``[{"datetime": "2024-02-05T08:00:00+01:00", value: 3.2}]`` + - ``[{"datetime": "2024-02-05T08:00:00+01:00", value: "3.2 kWh"}]`` - Exact set point(s) that the scheduler needs to realize (defaults to NaN values). * - ``soc-gain`` - ``.1kWh`` From 35ab5d07ac237987b7d7b07f674653adc22af2f0 Mon Sep 17 00:00:00 2001 From: Victor Garcia Reolid Date: Tue, 30 Jul 2024 17:38:34 +0200 Subject: [PATCH 29/47] feat: convert schedule results ffrom MW to sensor unit Signed-off-by: Victor Garcia Reolid --- flexmeasures/conftest.py | 13 +++ flexmeasures/data/models/planning/storage.py | 4 +- .../data/models/planning/tests/test_solver.py | 91 ++++++++++++++++++- 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index d7a3cf4f1..c7795ded2 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -819,6 +819,19 @@ def create_test_battery_assets( ) db.session.add(test_battery_sensor) + test_battery_sensor_kw = Sensor( + name="power (kW)", + generic_asset=test_battery, + event_resolution=timedelta(minutes=15), + unit="kW", + attributes=dict( + daily_seasonality=True, + weekly_seasonality=True, + yearly_seasonality=True, + ), + ) + db.session.add(test_battery_sensor_kw) + test_battery_no_prices = GenericAsset( name="Test battery with no known prices", owner=setup_accounts["Prosumer"], diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 1bd279c51..87a0b691b 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -27,7 +27,7 @@ from flexmeasures.data.schemas.scheduling import FlexContextSchema from flexmeasures.utils.time_utils import get_max_planning_horizon from flexmeasures.utils.coding_utils import deprecated -from flexmeasures.utils.unit_utils import ur +from flexmeasures.utils.unit_utils import ur, convert_units class MetaStorageScheduler(Scheduler): @@ -592,6 +592,7 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: storage_schedule = fallback_charging_policy( sensor, device_constraints[0], start, end, resolution ) + storage_schedule = convert_units(storage_schedule, "MW", sensor.unit) # Round schedule if self.round_to_decimals: @@ -649,6 +650,7 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: # Obtain the storage schedule from all device schedules within the EMS storage_schedule = ems_schedule[0] + storage_schedule = convert_units(storage_schedule, "MW", sensor.unit) # Round schedule if self.round_to_decimals: diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 3c7f1a81a..2e11f73a6 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -26,7 +26,7 @@ integrate_time_series, ) from flexmeasures.tests.utils import get_test_sensor -from flexmeasures.utils.unit_utils import convert_units +from flexmeasures.utils.unit_utils import convert_units, ur TOLERANCE = 0.00001 @@ -1019,10 +1019,14 @@ def compute_schedule(flex_model): compute_schedule(flex_model) -def get_sensors_from_db(db, battery_assets, battery_name="Test battery"): +def get_sensors_from_db( + db, battery_assets, battery_name="Test battery", power_sensor_name="power" +): # get the sensors from the database epex_da = get_test_sensor(db) - battery = battery_assets[battery_name].sensors[0] + battery = [ + s for s in battery_assets[battery_name].sensors if s.name == power_sensor_name + ][0] assert battery.get_attribute("market_id") == epex_da.id return epex_da, battery @@ -2041,3 +2045,84 @@ def compute_schedule(flex_model): # this yields the same results as with the SOC targets # because soc-maxima = soc-minima = soc-targets assert all(abs(soc[9:].values - values[:-1]) < 1e-5) + + +@pytest.mark.parametrize("unit", [None, "MWh", "kWh"]) +@pytest.mark.parametrize("soc_unit", ["kWh", "MWh"]) +@pytest.mark.parametrize("power_sensor_name", ["power", "power (kW)"]) +def test_battery_storage_different_units( + add_battery_assets, + db, + power_sensor_name, + soc_unit, + unit, +): + """ + Test scheduling a 1 MWh battery for 2h with a low -> high price transition with + different units for the soc-min, soc-max, soc-at-start and power sensor. + """ + + soc_min = ur.Quantity("100 kWh") + soc_max = ur.Quantity("1 MWh") + soc_at_start = ur.Quantity("100 kWh") + + if unit is not None: + soc_min = str(soc_min.to(unit)) + soc_max = str(soc_max.to(unit)) + soc_at_start = str(soc_at_start.to(unit)) + else: + soc_min = soc_min.to(soc_unit).magnitude + soc_max = soc_max.to(soc_unit).magnitude + soc_at_start = soc_at_start.to(soc_unit).magnitude + + epex_da, battery = get_sensors_from_db( + db, + add_battery_assets, + battery_name="Test battery", + power_sensor_name=power_sensor_name, + ) + tz = pytz.timezone("Europe/Amsterdam") + + # transition from cheap to expensive (90 -> 100) + start = tz.localize(datetime(2015, 1, 2, 14, 0, 0)) + end = tz.localize(datetime(2015, 1, 2, 16, 0, 0)) + resolution = timedelta(minutes=15) + + flex_model = { + "soc-min": soc_min, + "soc-max": soc_max, + "soc-at-start": soc_at_start, + "soc-unit": soc_unit, + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "power-capacity": "1 MW", + } + + scheduler: Scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model=flex_model, + flex_context={ + "site-power-capacity": "1 MW", + }, + ) + schedule = scheduler.compute() + + if power_sensor_name == "power (kW)": + schedule /= 1000 + + # charge fully in the cheap price period (100 kWh -> 1000kWh) + assert schedule[:4].sum() * 0.25 == 0.9 + + # discharge fully in the expensive price period (1000 kWh -> 100 kWh) + assert schedule[4:].sum() * 0.25 == -0.9 + + if isinstance(soc_at_start, str): + soc_at_start = ur.Quantity(soc_at_start).to("MWh").magnitude + elif isinstance(soc_at_start, float) or isinstance(soc_at_start, int): + soc_at_start = soc_at_start * convert_units(1, soc_unit, "MWh") + + # Check if constraints were met + check_constraints(battery, schedule, soc_at_start) From 2202077dfc53eaa869b8da744a4965c7a2686918 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 6 Aug 2024 09:57:09 +0200 Subject: [PATCH 30/47] feat: minimum list length for soc-gain and soc-usage Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/storage.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 8d41d6bf1..97efca7ff 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -139,10 +139,16 @@ class StorageFlexModelSchema(Schema): prefer_charging_sooner = fields.Bool(data_key="prefer-charging-sooner") soc_gain = fields.List( - TimeSeriesOrQuantityOrSensor("MW"), data_key="soc-gain", required=False + TimeSeriesOrQuantityOrSensor("MW"), + data_key="soc-gain", + required=False, + validate=validate.Length(min=1), ) soc_usage = fields.List( - TimeSeriesOrQuantityOrSensor("MW"), data_key="soc-usage", required=False + TimeSeriesOrQuantityOrSensor("MW"), + data_key="soc-usage", + required=False, + validate=validate.Length(min=1), ) def __init__( From 71ca8c4d6fe6f8daa3446146c1467b9fa2c3a529 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 6 Aug 2024 09:59:33 +0200 Subject: [PATCH 31/47] refactor: apply default SoC unit to all fields starting with "soc_" Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/storage.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 97efca7ff..d8a6874c2 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -185,12 +185,9 @@ def __init__( super().__init__(*args, **kwargs) if default_soc_unit is not None: - setattr(self.fields["soc_at_start"], "default_src_unit", default_soc_unit) - setattr(self.fields["soc_min"], "default_src_unit", default_soc_unit) - setattr(self.fields["soc_max"], "default_src_unit", default_soc_unit) - setattr(self.fields["soc_minima"], "default_src_unit", default_soc_unit) - setattr(self.fields["soc_maxima"], "default_src_unit", default_soc_unit) - setattr(self.fields["soc_targets"], "default_src_unit", default_soc_unit) + for field in self.fields.keys(): + if field.startswith("soc_"): + setattr(self.fields[field], "default_src_unit", default_soc_unit) @validates_schema def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs): From 8cfe9fdf4d087a3a15975c5004e372c76273385a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 6 Aug 2024 10:58:20 +0200 Subject: [PATCH 32/47] refactor: move warning Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index c9ca95291..2f19d921f 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -259,6 +259,10 @@ def __init__( self.to_unit = ur.Quantity(to_unit) self.default_src_unit = default_src_unit self.return_magnitude = return_magnitude + if self.return_magnitude is True: + current_app.logger.warning( + "Deserialized time series will include Quantity objects in the future. Set `return_magnitude=False` to trigger the new behaviour." + ) @with_appcontext_if_needed() def _deserialize( @@ -276,10 +280,6 @@ def _deserialize( return sensor elif isinstance(value, list): - if self.return_magnitude is True: - current_app.logger.warning( - "Deserialized time series will include Quantity objects in the future. Set `return_magnitude=False` to trigger the new behaviour." - ) field = fields.List( fields.Nested( TimedEventSchema( From ed56897406ee62b151520cd6c587da607c881aac Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 6 Aug 2024 13:29:14 +0200 Subject: [PATCH 33/47] Revert "refactor: move warning" This reverts commit 8cfe9fdf4d087a3a15975c5004e372c76273385a. --- flexmeasures/data/schemas/sensors.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 2f19d921f..c9ca95291 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -259,10 +259,6 @@ def __init__( self.to_unit = ur.Quantity(to_unit) self.default_src_unit = default_src_unit self.return_magnitude = return_magnitude - if self.return_magnitude is True: - current_app.logger.warning( - "Deserialized time series will include Quantity objects in the future. Set `return_magnitude=False` to trigger the new behaviour." - ) @with_appcontext_if_needed() def _deserialize( @@ -280,6 +276,10 @@ def _deserialize( return sensor elif isinstance(value, list): + if self.return_magnitude is True: + current_app.logger.warning( + "Deserialized time series will include Quantity objects in the future. Set `return_magnitude=False` to trigger the new behaviour." + ) field = fields.List( fields.Nested( TimedEventSchema( From df96eb9dbccaa26078441a2f28a595d0166da51a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 6 Aug 2024 15:03:59 +0200 Subject: [PATCH 34/47] fix: update test Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/tests/test_solver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 2e11f73a6..3fab349fe 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -1724,10 +1724,10 @@ def test_battery_stock_delta_sensor( "gain,usage,expected_delta", [ (["1 MW"], ["1MW"], 0), # delta stock is 0 (1 MW - 1 MW) - (["0.5 MW", "0.5 MW"], [], 1), # 1 MW stock gain + (["0.5 MW", "0.5 MW"], None, 1), # 1 MW stock gain (["100 kW"], None, 0.1), # 100 MW stock gain - (None, ["100 kW"], -0.1), # 100 kW stock loss - ([], [], None), # no gain defined -> no gain or loss happens + (None, ["100 kW"], -0.1), # 100 kW stock usage + (None, None, None), # no gain/usage defined -> no gain or usage happens ], ) def test_battery_stock_delta_quantity( @@ -1736,7 +1736,7 @@ def test_battery_stock_delta_quantity( """ Test the stock gain field when a constant value is provided. - We expect a constant gain/loss to happen in every time period equal to the energy + We expect a constant gain/usage to happen in every time period equal to the energy value provided. """ _, battery = get_sensors_from_db(db, add_battery_assets) From 88f259708e266e90b3c7e6cb5bcb54c4f0b615d8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 6 Aug 2024 15:52:55 +0200 Subject: [PATCH 35/47] refactor: move deserialization logic to dedicated class methods Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 86 ++++++++++++++++------------ 1 file changed, 49 insertions(+), 37 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index c9ca95291..77301d61a 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Any + from flask import current_app from marshmallow import ( Schema, @@ -266,51 +268,61 @@ def _deserialize( ) -> list[dict] | Sensor | ur.Quantity: if isinstance(value, dict): - if "sensor" not in value: - raise FMValidationError( - "Dictionary provided but `sensor` key not found." - ) - sensor = SensorIdField(unit=self.to_unit)._deserialize( - value["sensor"], None, None - ) - return sensor - + return self._deserialize_dict(value) elif isinstance(value, list): - if self.return_magnitude is True: - current_app.logger.warning( - "Deserialized time series will include Quantity objects in the future. Set `return_magnitude=False` to trigger the new behaviour." - ) - field = fields.List( - fields.Nested( - TimedEventSchema( - timezone=self.timezone, - value_validator=self.value_validator, - to_unit=self.to_unit, - default_src_unit=self.default_src_unit, - return_magnitude=self.return_magnitude, - ) - ) - ) - return field._deserialize(value, None, None) - + return self._deserialize_list(value) elif isinstance(value, str): - try: - return ur.Quantity(value).to(self.to_unit) - except DimensionalityError as e: - raise FMValidationError( - f"Cannot convert value `{value}` to '{self.to_unit}'" - ) from e - + return self._deserialize_str(value) elif self.default_src_unit is not None: - return self._deserialize( - f"{value} {self.default_src_unit}", attr, obj, **kwargs - ) - + return self._deserialize_numeric(value, attr, obj, **kwargs) else: raise FMValidationError( f"Unsupported value type. `{type(value)}` was provided but only dict, list and str are supported." ) + def _deserialize_dict(self, value: dict[str, int]) -> Sensor: + """Deserialize a sensor reference to a Sensor.""" + if "sensor" not in value: + raise FMValidationError("Dictionary provided but `sensor` key not found.") + sensor = SensorIdField(unit=self.to_unit)._deserialize( + value["sensor"], None, None + ) + return sensor + + def _deserialize_list(self, value: list[dict]) -> list[dict]: + """Deserialize a time series to a list of timed events.""" + if self.return_magnitude is True: + current_app.logger.warning( + "Deserialized time series will include Quantity objects in the future. Set `return_magnitude=False` to trigger the new behaviour." + ) + field = fields.List( + fields.Nested( + TimedEventSchema( + timezone=self.timezone, + value_validator=self.value_validator, + to_unit=self.to_unit, + default_src_unit=self.default_src_unit, + return_magnitude=self.return_magnitude, + ) + ) + ) + return field._deserialize(value, None, None) + + def _deserialize_str(self, value: str) -> ur.Quantity: + """Deserialize a string to a Quantity.""" + try: + return ur.Quantity(value).to(self.to_unit) + except DimensionalityError as e: + raise FMValidationError( + f"Cannot convert value `{value}` to '{self.to_unit}'" + ) from e + + def _deserialize_numeric(self, value: Any, attr, obj, **kwargs) -> ur.Quantity: + """Try to deserialize any other value (e.g. numeric) to a Quantity, using the default_src_unit.""" + return self._deserialize( + f"{value} {self.default_src_unit}", attr, obj, **kwargs + ) + def _serialize( self, value: ur.Quantity | Sensor | pd.Series, attr, data, **kwargs ) -> str | dict[str, int]: From 0acca6dc1d5be0f7a2296bcde8d1b84caad17f72 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 6 Aug 2024 17:07:00 +0200 Subject: [PATCH 36/47] docs: fix comment about converting time series using to_unit Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 77301d61a..9efa11273 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -241,9 +241,8 @@ def __init__( For example, validate=validate.Range(min=0) will raise a ValidationError in case of negative quantities, but will let pass any sensor that has recorded negative values. - :param to_unit: Unit in which the sensor or quantity should be convertible to. - - Time series are assumed to be passed without a unit, so they aren't checked for convertibility. - - Quantities will already be converted to the given unit. + :param to_unit: Unit in which the time series, quantity or sensor should be convertible to. + - Time series and quantities are converted to the given unit. - Sensors are checked for convertibility, but the original sensor is returned, so its values are not yet converted. :param default_src_unit: What unit to use in case of getting a numeric value. Does not apply to time series or sensors. :param return_magnitude: In case of getting a time series, whether the result should include the magnitude of each quantity, or each Quantity object itself From ab30487d24ef2b20884430ab2c902f937e9e45c3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 6 Aug 2024 17:12:00 +0200 Subject: [PATCH 37/47] refactor: rename new schema Signed-off-by: F.N. Claessen --- flexmeasures/cli/data_add.py | 26 ++++++++-------- flexmeasures/data/schemas/__init__.py | 2 +- .../data/schemas/scheduling/__init__.py | 8 ++--- .../data/schemas/scheduling/storage.py | 30 +++++++++---------- flexmeasures/data/schemas/sensors.py | 18 ++++++----- .../data/schemas/tests/test_sensor.py | 4 +-- 6 files changed, 45 insertions(+), 43 deletions(-) diff --git a/flexmeasures/cli/data_add.py b/flexmeasures/cli/data_add.py index 532dee37e..e959de470 100755 --- a/flexmeasures/cli/data_add.py +++ b/flexmeasures/cli/data_add.py @@ -62,7 +62,7 @@ LongitudeField, SensorIdField, TimeIntervalField, - TimeSeriesOrQuantityOrSensor, + VariableQuantityField, ) from flexmeasures.data.schemas.sources import DataSourceIdField from flexmeasures.data.schemas.times import TimeIntervalSchema @@ -1123,7 +1123,7 @@ def create_schedule(ctx): @click.option( "--site-power-capacity", "site_power_capacity", - type=TimeSeriesOrQuantityOrSensor("MW"), + type=VariableQuantityField("MW"), required=False, default=None, help="Site consumption/production power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1133,7 +1133,7 @@ def create_schedule(ctx): @click.option( "--site-consumption-capacity", "site_consumption_capacity", - type=TimeSeriesOrQuantityOrSensor("MW"), + type=VariableQuantityField("MW"), required=False, default=None, help="Site consumption power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1143,7 +1143,7 @@ def create_schedule(ctx): @click.option( "--site-production-capacity", "site_production_capacity", - type=TimeSeriesOrQuantityOrSensor("MW"), + type=VariableQuantityField("MW"), required=False, default=None, help="Site production power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1208,7 +1208,7 @@ def create_schedule(ctx): @click.option( "--charging-efficiency", "charging_efficiency", - type=TimeSeriesOrQuantityOrSensor("%"), + type=VariableQuantityField("%"), required=False, default=None, help="Storage charging efficiency to use for the schedule." @@ -1218,7 +1218,7 @@ def create_schedule(ctx): @click.option( "--discharging-efficiency", "discharging_efficiency", - type=TimeSeriesOrQuantityOrSensor("%"), + type=VariableQuantityField("%"), required=False, default=None, help="Storage discharging efficiency to use for the schedule." @@ -1228,7 +1228,7 @@ def create_schedule(ctx): @click.option( "--soc-gain", "soc_gain", - type=TimeSeriesOrQuantityOrSensor("MW"), + type=VariableQuantityField("MW"), required=False, default=None, help="Specify the State of Charge (SoC) gain as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1238,7 +1238,7 @@ def create_schedule(ctx): @click.option( "--soc-usage", "soc_usage", - type=TimeSeriesOrQuantityOrSensor("MW"), + type=VariableQuantityField("MW"), required=False, default=None, help="Specify the State of Charge (SoC) usage as a quantity in power units (e.g. 1 MW or 1000 kW) " @@ -1248,7 +1248,7 @@ def create_schedule(ctx): @click.option( "--storage-power-capacity", "storage_power_capacity", - type=TimeSeriesOrQuantityOrSensor("MW"), + type=VariableQuantityField("MW"), required=False, default=None, help="Storage consumption/production power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1258,7 +1258,7 @@ def create_schedule(ctx): @click.option( "--storage-consumption-capacity", "storage_consumption_capacity", - type=TimeSeriesOrQuantityOrSensor("MW"), + type=VariableQuantityField("MW"), required=False, default=None, help="Storage consumption power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1268,7 +1268,7 @@ def create_schedule(ctx): @click.option( "--storage-production-capacity", "storage_production_capacity", - type=TimeSeriesOrQuantityOrSensor("MW"), + type=VariableQuantityField("MW"), required=False, default=None, help="Storage production power capacity. Provide this as a quantity in power units (e.g. 1 MW or 1000 kW)" @@ -1278,7 +1278,7 @@ def create_schedule(ctx): @click.option( "--storage-efficiency", "storage_efficiency", - type=TimeSeriesOrQuantityOrSensor("%", default_src_unit="dimensionless"), + type=VariableQuantityField("%", default_src_unit="dimensionless"), required=False, default="100%", help="Storage efficiency (e.g. 95% or 0.95) to use for the schedule," @@ -1421,7 +1421,7 @@ def add_schedule_for_storage( # noqa C901 else: unit = "MW" - scheduling_kwargs[key][field_name] = TimeSeriesOrQuantityOrSensor( + scheduling_kwargs[key][field_name] = VariableQuantityField( unit )._serialize(value, None, None) diff --git a/flexmeasures/data/schemas/__init__.py b/flexmeasures/data/schemas/__init__.py index dbf4ff7b3..8f56be0ab 100644 --- a/flexmeasures/data/schemas/__init__.py +++ b/flexmeasures/data/schemas/__init__.py @@ -5,7 +5,7 @@ from .account import AccountIdField from .generic_assets import GenericAssetIdField as AssetIdField from .locations import LatitudeField, LongitudeField -from .sensors import SensorIdField, TimeSeriesOrQuantityOrSensor +from .sensors import SensorIdField, VariableQuantityField from .sources import DataSourceIdField as SourceIdField from .times import ( AwareDateTimeField, diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 40915ceeb..b40915366 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -1,7 +1,7 @@ from marshmallow import Schema, fields, validate from flexmeasures.data.schemas.sensors import ( - TimeSeriesOrQuantityOrSensor, + VariableQuantityField, SensorIdField, ) @@ -11,19 +11,19 @@ class FlexContextSchema(Schema): This schema lists fields that can be used to describe sensors in the optimised portfolio """ - ems_power_capacity_in_mw = TimeSeriesOrQuantityOrSensor( + ems_power_capacity_in_mw = VariableQuantityField( "MW", required=False, data_key="site-power-capacity", validate=validate.Range(min=0), ) - ems_production_capacity_in_mw = TimeSeriesOrQuantityOrSensor( + ems_production_capacity_in_mw = VariableQuantityField( "MW", required=False, data_key="site-production-capacity", validate=validate.Range(min=0), ) - ems_consumption_capacity_in_mw = TimeSeriesOrQuantityOrSensor( + ems_consumption_capacity_in_mw = VariableQuantityField( "MW", required=False, data_key="site-consumption-capacity", diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index d8a6874c2..6976ad61d 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -15,7 +15,7 @@ from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.schemas.units import QuantityField -from flexmeasures.data.schemas.sensors import TimeSeriesOrQuantityOrSensor +from flexmeasures.data.schemas.sensors import VariableQuantityField from flexmeasures.utils.unit_utils import ur @@ -78,26 +78,26 @@ class StorageFlexModelSchema(Schema): data_key="soc-max", ) - power_capacity_in_mw = TimeSeriesOrQuantityOrSensor( + power_capacity_in_mw = VariableQuantityField( "MW", required=False, data_key="power-capacity" ) - consumption_capacity = TimeSeriesOrQuantityOrSensor( + consumption_capacity = VariableQuantityField( "MW", data_key="consumption-capacity", required=False ) - production_capacity = TimeSeriesOrQuantityOrSensor( + production_capacity = VariableQuantityField( "MW", data_key="production-capacity", required=False ) # Timezone placeholders for the soc_maxima, soc_minima and soc_targets fields are overridden in __init__ - soc_maxima = TimeSeriesOrQuantityOrSensor( + soc_maxima = VariableQuantityField( to_unit="MWh", default_src_unit="dimensionless", # placeholder, overridden in __init__ timezone="placeholder", data_key="soc-maxima", ) - soc_minima = TimeSeriesOrQuantityOrSensor( + soc_minima = VariableQuantityField( to_unit="MWh", default_src_unit="dimensionless", # placeholder, overridden in __init__ timezone="placeholder", @@ -105,7 +105,7 @@ class StorageFlexModelSchema(Schema): value_validator=validate.Range(min=0), ) - soc_targets = TimeSeriesOrQuantityOrSensor( + soc_targets = VariableQuantityField( to_unit="MWh", default_src_unit="dimensionless", # placeholder, overridden in __init__ timezone="placeholder", @@ -122,10 +122,10 @@ class StorageFlexModelSchema(Schema): data_key="soc-unit", ) - charging_efficiency = TimeSeriesOrQuantityOrSensor( + charging_efficiency = VariableQuantityField( "%", data_key="charging-efficiency", required=False ) - discharging_efficiency = TimeSeriesOrQuantityOrSensor( + discharging_efficiency = VariableQuantityField( "%", data_key="discharging-efficiency", required=False ) @@ -133,19 +133,19 @@ class StorageFlexModelSchema(Schema): data_key="roundtrip-efficiency", required=False ) - storage_efficiency = TimeSeriesOrQuantityOrSensor( + storage_efficiency = VariableQuantityField( "%", default_src_unit="dimensionless", data_key="storage-efficiency" ) prefer_charging_sooner = fields.Bool(data_key="prefer-charging-sooner") soc_gain = fields.List( - TimeSeriesOrQuantityOrSensor("MW"), + VariableQuantityField("MW"), data_key="soc-gain", required=False, validate=validate.Length(min=1), ) soc_usage = fields.List( - TimeSeriesOrQuantityOrSensor("MW"), + VariableQuantityField("MW"), data_key="soc-usage", required=False, validate=validate.Length(min=1), @@ -162,21 +162,21 @@ def __init__( """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" self.start = start self.sensor = sensor - self.soc_maxima = TimeSeriesOrQuantityOrSensor( + self.soc_maxima = VariableQuantityField( to_unit="MWh", default_src_unit=default_soc_unit, timezone=sensor.timezone, data_key="soc-maxima", ) - self.soc_minima = TimeSeriesOrQuantityOrSensor( + self.soc_minima = VariableQuantityField( to_unit="MWh", default_src_unit=default_soc_unit, timezone=sensor.timezone, data_key="soc-minima", value_validator=validate.Range(min=0), ) - self.soc_targets = TimeSeriesOrQuantityOrSensor( + self.soc_targets = VariableQuantityField( to_unit="MWh", default_src_unit=default_soc_unit, timezone=sensor.timezone, diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 9efa11273..95ad5b6f6 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -222,7 +222,7 @@ def _serialize(self, sensor: Sensor, attr, data, **kwargs) -> int: return sensor.id -class TimeSeriesOrQuantityOrSensor(MarshmallowClickMixin, fields.Field): +class VariableQuantityField(MarshmallowClickMixin, fields.Field): def __init__( self, to_unit, @@ -233,7 +233,9 @@ def __init__( value_validator: Validator | None = None, **kwargs, ): - """Field for validating, serializing and deserializing a quantity, sensor or time series. + """Field for validating, serializing and deserializing a variable quantity. + + A variable quantity can be represented by a fixed quantity, sensor or time series. # todo: Sensor should perhaps deserialize already to sensor data @@ -366,19 +368,19 @@ def __call__(self, value): return value -class QuantityOrSensor(TimeSeriesOrQuantityOrSensor): +class QuantityOrSensor(VariableQuantityField): def __init__(self, *args, **kwargs): - """Deprecated class. Use `TimeSeriesOrQuantityOrSensor` instead.""" + """Deprecated class. Use `VariableQuantityField` instead.""" current_app.logger.warning( - "Class `TimeSeriesOrSensor` is deprecated. Use `TimeSeriesOrQuantityOrSensor` instead." + "Class `TimeSeriesOrSensor` is deprecated. Use `VariableQuantityField` instead." ) super().__init__(*args, **kwargs) -class TimeSeriesOrSensor(TimeSeriesOrQuantityOrSensor): +class TimeSeriesOrSensor(VariableQuantityField): def __init__(self, *args, **kwargs): - """Deprecated class. Use `TimeSeriesOrQuantityOrSensor` instead.""" + """Deprecated class. Use `VariableQuantityField` instead.""" current_app.logger.warning( - "Class `TimeSeriesOrSensor` is deprecated. Use `TimeSeriesOrQuantityOrSensor` instead." + "Class `TimeSeriesOrSensor` is deprecated. Use `VariableQuantityField` instead." ) super().__init__(*args, **kwargs) diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index ff7335c97..4acd712cb 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -2,7 +2,7 @@ from flexmeasures import Sensor from flexmeasures.data.schemas.sensors import ( QuantityOrSensor, - TimeSeriesOrQuantityOrSensor, + VariableQuantityField, ) from flexmeasures.utils.unit_utils import ur from marshmallow import ValidationError @@ -112,7 +112,7 @@ def test_quantity_or_sensor_field( ) def test_time_series_field(input_param, dst_unit, fails, db): - field = TimeSeriesOrQuantityOrSensor( + field = VariableQuantityField( to_unit=dst_unit, default_src_unit="MWh", return_magnitude=False, From 8d17a9296a410446a6a7eb913e1c4a326e2bdbdf Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 6 Aug 2024 17:16:47 +0200 Subject: [PATCH 38/47] refactor: sync code order: 1) Sensor, 2) time series, 3) Quantity Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 95ad5b6f6..64b6d92c0 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -235,7 +235,7 @@ def __init__( ): """Field for validating, serializing and deserializing a variable quantity. - A variable quantity can be represented by a fixed quantity, sensor or time series. + A variable quantity can be represented by a sensor, time series or fixed quantity. # todo: Sensor should perhaps deserialize already to sensor data @@ -243,9 +243,9 @@ def __init__( For example, validate=validate.Range(min=0) will raise a ValidationError in case of negative quantities, but will let pass any sensor that has recorded negative values. - :param to_unit: Unit in which the time series, quantity or sensor should be convertible to. - - Time series and quantities are converted to the given unit. + :param to_unit: Unit in which the sensor, time series or quantity should be convertible to. - Sensors are checked for convertibility, but the original sensor is returned, so its values are not yet converted. + - Time series and quantities are already converted to the given unit. :param default_src_unit: What unit to use in case of getting a numeric value. Does not apply to time series or sensors. :param return_magnitude: In case of getting a time series, whether the result should include the magnitude of each quantity, or each Quantity object itself :param timezone: Only used in case a time series is specified and one of the *timed events* @@ -266,7 +266,7 @@ def __init__( @with_appcontext_if_needed() def _deserialize( self, value: dict[str, int] | list[dict] | str, attr, obj, **kwargs - ) -> list[dict] | Sensor | ur.Quantity: + ) -> Sensor | list[dict] | ur.Quantity: if isinstance(value, dict): return self._deserialize_dict(value) @@ -325,16 +325,16 @@ def _deserialize_numeric(self, value: Any, attr, obj, **kwargs) -> ur.Quantity: ) def _serialize( - self, value: ur.Quantity | Sensor | pd.Series, attr, data, **kwargs + self, value: Sensor | pd.Series | ur.Quantity, attr, data, **kwargs ) -> str | dict[str, int]: - if isinstance(value, ur.Quantity): - return str(value.to(self.to_unit)) - elif isinstance(value, Sensor): + if isinstance(value, Sensor): return dict(sensor=value.id) elif isinstance(value, pd.Series): raise NotImplementedError( "Serialization of a time series from a Pandas Series is not implemented yet." ) + elif isinstance(value, ur.Quantity): + return str(value.to(self.to_unit)) else: raise FMValidationError( "Serialized quantity, sensor or time series needs to be of type int, float, Sensor or pandas.Series." From 161f31463b63b02603cf52c1d02b3f4b9939878d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 8 Aug 2024 22:55:38 +0200 Subject: [PATCH 39/47] fix: add test and add missing logic for handling a list of dictionaries representing a time series Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 26 ++-- .../data/models/planning/tests/test_solver.py | 118 +++++++++++++++++- flexmeasures/data/models/planning/utils.py | 42 ++++--- 3 files changed, 153 insertions(+), 33 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 87a0b691b..6b84c7da1 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -117,7 +117,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 power_capacity_in_mw = ur.Quantity(f"{power_capacity_in_mw} MW") power_capacity_in_mw = get_continuous_series_sensor_or_quantity( - quantity_or_sensor=power_capacity_in_mw, + variable_quantity=power_capacity_in_mw, actuator=sensor, unit="MW", query_window=(start, end), @@ -183,7 +183,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # fetch SOC constraints from sensors if isinstance(soc_targets, Sensor): soc_targets = get_continuous_series_sensor_or_quantity( - quantity_or_sensor=soc_targets, + variable_quantity=soc_targets, actuator=sensor, unit="MWh", query_window=(start, end), @@ -194,7 +194,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 ) if isinstance(soc_minima, Sensor): soc_minima = get_continuous_series_sensor_or_quantity( - quantity_or_sensor=soc_minima, + variable_quantity=soc_minima, actuator=sensor, unit="MWh", query_window=(start, end), @@ -205,7 +205,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 ) if isinstance(soc_maxima, Sensor): soc_maxima = get_continuous_series_sensor_or_quantity( - quantity_or_sensor=soc_maxima, + variable_quantity=soc_maxima, actuator=sensor, unit="MWh", query_window=(start, end), @@ -236,7 +236,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[0]["derivative min"] = ( -1 ) * get_continuous_series_sensor_or_quantity( - quantity_or_sensor=production_capacity, + variable_quantity=production_capacity, actuator=sensor, unit="MW", query_window=(start, end), @@ -251,7 +251,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[0][ "derivative max" ] = get_continuous_series_sensor_or_quantity( - quantity_or_sensor=consumption_capacity, + variable_quantity=consumption_capacity, actuator=sensor, unit="MW", query_window=(start, end), @@ -269,7 +269,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 for is_usage, soc_delta in zip([False, True], [soc_gain, soc_usage]): for component in soc_delta: stock_delta_series = get_continuous_series_sensor_or_quantity( - quantity_or_sensor=component, + variable_quantity=component, actuator=sensor, unit="MW", query_window=(start, end), @@ -295,7 +295,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # Apply round-trip efficiency evenly to charging and discharging charging_efficiency = get_continuous_series_sensor_or_quantity( - quantity_or_sensor=self.flex_model.get("charging_efficiency"), + variable_quantity=self.flex_model.get("charging_efficiency"), actuator=sensor, unit="dimensionless", query_window=(start, end), @@ -304,7 +304,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 fallback_attribute="charging-efficiency", ).fillna(1) discharging_efficiency = get_continuous_series_sensor_or_quantity( - quantity_or_sensor=self.flex_model.get("discharging_efficiency"), + variable_quantity=self.flex_model.get("discharging_efficiency"), actuator=sensor, unit="dimensionless", query_window=(start, end), @@ -333,7 +333,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 ): device_constraints[0]["efficiency"] = ( get_continuous_series_sensor_or_quantity( - quantity_or_sensor=storage_efficiency, + variable_quantity=storage_efficiency, actuator=sensor, unit="dimensionless", query_window=(start, end), @@ -372,7 +372,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 ) ems_power_capacity_in_mw = get_continuous_series_sensor_or_quantity( - quantity_or_sensor=self.flex_context.get("ems_power_capacity_in_mw"), + variable_quantity=self.flex_context.get("ems_power_capacity_in_mw"), actuator=sensor.generic_asset, unit="MW", query_window=(start, end), @@ -382,7 +382,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 ) ems_constraints["derivative max"] = get_continuous_series_sensor_or_quantity( - quantity_or_sensor=self.flex_context.get("ems_consumption_capacity_in_mw"), + variable_quantity=self.flex_context.get("ems_consumption_capacity_in_mw"), actuator=sensor.generic_asset, unit="MW", query_window=(start, end), @@ -394,7 +394,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 ems_constraints["derivative min"] = ( -1 ) * get_continuous_series_sensor_or_quantity( - quantity_or_sensor=self.flex_context.get("ems_production_capacity_in_mw"), + variable_quantity=self.flex_context.get("ems_production_capacity_in_mw"), actuator=sensor.generic_asset, unit="MW", query_window=(start, end), diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 3fab349fe..115f317b1 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -2113,16 +2113,126 @@ def test_battery_storage_different_units( if power_sensor_name == "power (kW)": schedule /= 1000 + # Check if constraints were met + if isinstance(soc_at_start, str): + soc_at_start = ur.Quantity(soc_at_start).to("MWh").magnitude + elif isinstance(soc_at_start, float) or isinstance(soc_at_start, int): + soc_at_start = soc_at_start * convert_units(1, soc_unit, "MWh") + check_constraints(battery, schedule, soc_at_start) + # charge fully in the cheap price period (100 kWh -> 1000kWh) assert schedule[:4].sum() * 0.25 == 0.9 # discharge fully in the expensive price period (1000 kWh -> 100 kWh) assert schedule[4:].sum() * 0.25 == -0.9 - if isinstance(soc_at_start, str): - soc_at_start = ur.Quantity(soc_at_start).to("MWh").magnitude - elif isinstance(soc_at_start, float) or isinstance(soc_at_start, int): - soc_at_start = soc_at_start * convert_units(1, soc_unit, "MWh") + +@pytest.mark.parametrize( + "ts_field, ts_specs", + [ + # The battery only has time to charge up to 950 kWh halfway + ( + "power-capacity", + [ + { + "start": "2015-01-02T14:00+01", + "end": "2015-01-02T16:00+01", + "value": "850 kW", + } + ], + ), + # Same, but the event time is specified with a duration instead of an end time + ( + "power-capacity", + [ + { + "start": "2015-01-02T14:00+01", + "duration": "PT2H", + "value": "850 kW", + } + ], + ), + # Can only charge up to 950 kWh halfway + ( + "soc-maxima", + [ + { + "datetime": "2015-01-02T15:00+01", + "value": "950 kWh", + } + ], + ), + # Must end up at a minimum of 200 kWh, for which it is cheapest to charge to 950 and then to discharge to 200 + ( + "soc-minima", + [ + { + "datetime": "2015-01-02T16:00+01", + "value": "200 kWh", + } + ], + ), + ], +) +def test_battery_storage_with_time_series_in_flex_model( + add_battery_assets, + db, + ts_field, + ts_specs, +): + """ + Test scheduling a 1 MWh battery for 2h with a low -> high price transition with + a time series used for the various flex-model fields. + """ + + soc_min = "100 kWh" + soc_max = "1 MWh" + soc_at_start = "100 kWh" + + epex_da, battery = get_sensors_from_db( + db, + add_battery_assets, + battery_name="Test battery", + power_sensor_name="power", + ) + tz = pytz.timezone("Europe/Amsterdam") + + # transition from cheap to expensive (90 -> 100) + start = tz.localize(datetime(2015, 1, 2, 14, 0, 0)) + end = tz.localize(datetime(2015, 1, 2, 16, 0, 0)) + resolution = timedelta(minutes=15) + + flex_model = { + "soc-min": soc_min, + "soc-max": soc_max, + "soc-at-start": soc_at_start, + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "power-capacity": "1 MW", + } + flex_model[ts_field] = ts_specs + + scheduler: Scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model=flex_model, + flex_context={ + "site-power-capacity": "1 MW", + }, + ) + schedule = scheduler.compute() # Check if constraints were met + soc_at_start = ur.Quantity(soc_at_start).to("MWh").magnitude check_constraints(battery, schedule, soc_at_start) + + # charge 850 kWh in the cheap price period (100 kWh -> 950kWh) + assert schedule[:4].sum() * 0.25 == pytest.approx(0.85) + + # discharge fully or to what's needed in the expensive price period (950 kWh -> 100 or 200 kWh) + if ts_field == "soc-minima": + assert schedule[4:].sum() * 0.25 == pytest.approx(-0.75) + else: + assert schedule[4:].sum() * 0.25 == pytest.approx(-0.85) diff --git a/flexmeasures/data/models/planning/utils.py b/flexmeasures/data/models/planning/utils.py index 908cf65e6..7e6b85a79 100644 --- a/flexmeasures/data/models/planning/utils.py +++ b/flexmeasures/data/models/planning/utils.py @@ -326,7 +326,7 @@ def get_quantity_from_attribute( def get_series_from_quantity_or_sensor( - quantity_or_sensor: Sensor | ur.Quantity, + variable_quantity: Sensor | list[dict] | ur.Quantity, unit: ur.Quantity | str, query_window: tuple[datetime, datetime], resolution: timedelta, @@ -337,8 +337,11 @@ def get_series_from_quantity_or_sensor( """ Get a time series given a quantity or sensor defined on a time window. - :param quantity_or_sensor: A pint Quantity or timely-beliefs Sensor, measuring e.g. power capacity - or efficiency. + :param variable_quantity: Variable quantity measuring e.g. power capacity or efficiency. + One of the following types: + - a timely-beliefs Sensor recording the data + - a list of dictionaries representing a time series specification + - a pint Quantity representing a fixed quantity :param unit: Unit of the output data. :param query_window: Tuple representing the start and end of the requested data. :param resolution: Time resolution of the requested data. @@ -352,15 +355,15 @@ def get_series_from_quantity_or_sensor( start, end = query_window index = initialize_index(start=start, end=end, resolution=resolution) - if isinstance(quantity_or_sensor, ur.Quantity): - if np.isnan(quantity_or_sensor.magnitude): + if isinstance(variable_quantity, ur.Quantity): + if np.isnan(variable_quantity.magnitude): magnitude = np.nan else: - magnitude = quantity_or_sensor.to(unit).magnitude + magnitude = variable_quantity.to(unit).magnitude time_series = pd.Series(magnitude, index=index) - elif isinstance(quantity_or_sensor, Sensor): + elif isinstance(variable_quantity, Sensor): bdf: tb.BeliefsDataFrame = TimedBelief.search( - quantity_or_sensor, + variable_quantity, event_starts_after=query_window[0], event_ends_before=query_window[1], resolution=resolution, @@ -372,18 +375,25 @@ def get_series_from_quantity_or_sensor( if as_instantaneous_events: bdf = bdf.resample_events(timedelta(0), boundary_policy=boundary_policy) time_series = simplify_index(bdf).reindex(index).squeeze() - time_series = convert_units(time_series, quantity_or_sensor.unit, unit) + time_series = convert_units(time_series, variable_quantity.unit, unit) + elif isinstance(variable_quantity, list): + time_series = pd.Series(np.nan, index=index) + for event in variable_quantity: + value = event["value"] + start = event["start"] + end = event["end"] + time_series[start : end - resolution] = value else: raise TypeError( - f"quantity_or_sensor {quantity_or_sensor} should be a pint Quantity or timely-beliefs Sensor" + f"quantity_or_sensor {variable_quantity} should be a pint Quantity or timely-beliefs Sensor" ) return time_series def get_continuous_series_sensor_or_quantity( - quantity_or_sensor: Sensor | ur.Quantity | None, + variable_quantity: Sensor | list[dict] | ur.Quantity | None, actuator: Sensor | Asset, unit: ur.Quantity | str, query_window: tuple[datetime, datetime], @@ -394,10 +404,10 @@ def get_continuous_series_sensor_or_quantity( as_instantaneous_events: bool = False, boundary_policy: str | None = None, ) -> pd.Series: - """Creates a time series from a quantity or sensor within a specified window, + """Creates a time series from a sensor, time series specification, or quantity within a specified window, falling back to a given `fallback_attribute` and making sure no values exceed `max_value`. - :param quantity_or_sensor: The quantity or sensor containing the data. + :param variable_quantity: A sensor recording the data, a time series specification or a fixed quantity. :param actuator: The actuator from which relevant defaults are retrieved. :param unit: The desired unit of the data. :param query_window: The time window (start, end) to query the data. @@ -409,15 +419,15 @@ def get_continuous_series_sensor_or_quantity( interpreted as the desired frequency of the data. :returns: time series data with missing values handled based on the chosen method. """ - if quantity_or_sensor is None: - quantity_or_sensor = get_quantity_from_attribute( + if variable_quantity is None: + variable_quantity = get_quantity_from_attribute( entity=actuator, attribute=fallback_attribute, unit=unit, ) time_series = get_series_from_quantity_or_sensor( - quantity_or_sensor=quantity_or_sensor, + variable_quantity=variable_quantity, unit=unit, query_window=query_window, resolution=resolution, From 03bac840b18c352f327e571fd1676824c83a75d2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 9 Aug 2024 16:10:54 +0200 Subject: [PATCH 40/47] fix: check for real numeric values Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/sensors.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 64b6d92c0..734b80f66 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any +import numbers from flask import current_app from marshmallow import ( @@ -274,7 +274,7 @@ def _deserialize( return self._deserialize_list(value) elif isinstance(value, str): return self._deserialize_str(value) - elif self.default_src_unit is not None: + elif isinstance(value, numbers.Real) and self.default_src_unit is not None: return self._deserialize_numeric(value, attr, obj, **kwargs) else: raise FMValidationError( @@ -318,8 +318,10 @@ def _deserialize_str(self, value: str) -> ur.Quantity: f"Cannot convert value `{value}` to '{self.to_unit}'" ) from e - def _deserialize_numeric(self, value: Any, attr, obj, **kwargs) -> ur.Quantity: - """Try to deserialize any other value (e.g. numeric) to a Quantity, using the default_src_unit.""" + def _deserialize_numeric( + self, value: numbers.Real, attr, obj, **kwargs + ) -> ur.Quantity: + """Try to deserialize a numeric value to a Quantity, using the default_src_unit.""" return self._deserialize( f"{value} {self.default_src_unit}", attr, obj, **kwargs ) From 1ffc3353f364a3997d16083d2104d82332c5c196 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 9 Aug 2024 16:53:29 +0200 Subject: [PATCH 41/47] docs: changelog entries Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 27 +++++++++++++++++++++++++++ documentation/changelog.rst | 1 + 2 files changed, 28 insertions(+) diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 440269e89..33da3fb87 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -6,6 +6,33 @@ API change log .. note:: The FlexMeasures API follows its own versioning scheme. This is also reflected in the URL (e.g. `/api/v3_0`), allowing developers to upgrade at their own pace. +v3.0-19 | 2024-08-09 +"""""""""""""""""""" + +- Allow setting a SoC unit directly in some fields (formerly ``Float`` fields, and now ``Quantity`` fields), while still falling back on the contents of the ``soc-unit`` field, for backwards compatibility: + + - ``soc-at-start`` + - ``soc-min`` + - ``soc-max`` + +- Allow setting a unit directly in fields that already supported passing a time series: + + - ``soc-maxima`` + - ``soc-minima`` + - ``soc-targets`` + +- Allow passing a time series in fields that formerly only accepted passing a fixed quantity or a sensor reference: + + - ``power-capacity`` + - ``consumption-capacity`` + - ``production-capacity`` + - ``charging-efficiency`` + - ``discharging-efficiency`` + - ``storage-efficiency`` + - ``soc-gain`` + - ``soc-usage`` + + v3.0-18 | 2024-03-07 """""""""""""""""""" - Add support for providing a sensor definition to the ``soc-minima``, ``soc-maxima`` and ``soc-targets`` flex-model fields for `/sensors//schedules/trigger` (POST). diff --git a/documentation/changelog.rst b/documentation/changelog.rst index f20f7aa42..953afcf56 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -8,6 +8,7 @@ v0.23.0 | August XX, 2024 New features ------------- +* Allow three ways of passing a variable quantity in most of the ``flex-model`` and ``flex-context`` fields [see `PR #1127 `_] Infrastructure / Support ---------------------- From 8ccc6f6c5f8ef16a197c804fdaeb3cb9eda8ba8e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 9 Aug 2024 16:54:28 +0200 Subject: [PATCH 42/47] docs: fix typo Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 953afcf56..ae6ce06b4 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -12,7 +12,7 @@ New features Infrastructure / Support ---------------------- -* Support new single-belief fast track (looking uup only one belief) [see `PR #1067 `_] +* Support new single-belief fast track (looking up only one belief) [see `PR #1067 `_] * Add new annotation types: ``"error"`` and ``"warning"`` [see `PR #1131 `_] * Removed deprecated ``app.schedulers`` and ``app.forecasters`` (use ``app.data_generators["scheduler"]`` and ``app.data_generators["forecaster"]`` instead) [see `PR #1098 `_] From 1a430c08cf121995967fd0386acc582f50228164 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 9 Aug 2024 17:00:25 +0200 Subject: [PATCH 43/47] style: be more explicit about requiring the soc-unit field to be set Signed-off-by: F.N. Claessen --- flexmeasures/data/schemas/scheduling/storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 6976ad61d..87c1f47df 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -120,6 +120,7 @@ class StorageFlexModelSchema(Schema): ] ), data_key="soc-unit", + required=False, ) charging_efficiency = VariableQuantityField( From 8ec14c59e5d0b4f62a79ed4c0def3db77cb063ca Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 16 Aug 2024 10:32:34 +0200 Subject: [PATCH 44/47] docs: mention the new Marshmallow field Signed-off-by: F.N. Claessen --- documentation/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index d45dee3b0..b56230446 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -11,7 +11,7 @@ New features ------------- * Add basic sensor info to sensor page [see `PR #1115 `_] * Support zoom-in action on the asset and sensor charts [see `PR #1130 `_] -* Allow three ways of passing a variable quantity in most of the ``flex-model`` and ``flex-context`` fields [see `PR #1127 `_] +* Introduce the ``VariableQuantityField`` to allow three ways of passing a variable quantity in most of the ``flex-model`` and ``flex-context`` fields [see `PR #1127 `_] Infrastructure / Support ---------------------- From 8fd4ec14743f045a7009be501705156e0d140ef7 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 16 Aug 2024 10:46:30 +0200 Subject: [PATCH 45/47] docs: advise setting a unit per field explicitly Signed-off-by: F.N. Claessen --- documentation/features/scheduling.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 153b53610..eed293092 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -113,6 +113,7 @@ and what constraints or preferences should be taken into account. * - ``soc-unit`` - ``"kWh"`` or ``"MWh"`` - The unit used to interpret any SoC related flex-model value that does not mention a unit itself (only applies to numeric values, so not to string values). + However, we advise to mention the unit in each field explicitly (for instance, ``"3.1 kWh"`` rather than ``3.1``). * - ``soc-min`` - ``"2.5 kWh"`` - A constant lower boundary for all values in the schedule (defaults to 0). From 15e19dcdea8b064cf11528eab90712d9db71900c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 16 Aug 2024 10:55:18 +0200 Subject: [PATCH 46/47] refactor: simplify if statement Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 6b84c7da1..060bf8ba5 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -457,7 +457,7 @@ def deserialize_flex_config(self): else: self.flex_model["soc-at-start"] = 0 # soc-unit - if "soc-unit" not in self.flex_model or self.flex_model["soc-unit"] is None: + if self.flex_model.get("soc-unit") is None: if self.sensor.unit in ("MWh", "kWh"): self.flex_model["soc-unit"] = self.sensor.unit elif self.sensor.unit in ("MW", "kW"): From 8c233ca07382031d749486975faf43eedf98342b Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 16 Aug 2024 15:15:48 +0200 Subject: [PATCH 47/47] refactor: move soc-unit guesswork into schema Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 11 +---------- flexmeasures/data/schemas/scheduling/storage.py | 8 ++++++++ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 060bf8ba5..bac503f65 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -456,15 +456,6 @@ def deserialize_flex_config(self): ) else: self.flex_model["soc-at-start"] = 0 - # soc-unit - if self.flex_model.get("soc-unit") is None: - if self.sensor.unit in ("MWh", "kWh"): - self.flex_model["soc-unit"] = self.sensor.unit - elif self.sensor.unit in ("MW", "kW"): - self.flex_model["soc-unit"] = self.sensor.unit + "h" - else: - # todo: raise? Surely we must have a unit. - pass self.ensure_soc_min_max() @@ -472,7 +463,7 @@ def deserialize_flex_config(self): self.flex_model = StorageFlexModelSchema( start=self.start, sensor=self.sensor, - default_soc_unit=self.flex_model["soc-unit"], + default_soc_unit=self.flex_model.get("soc-unit"), ).load(self.flex_model) self.flex_context = FlexContextSchema().load(self.flex_context) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 87c1f47df..4b3a5d612 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -163,6 +163,14 @@ def __init__( """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" self.start = start self.sensor = sensor + + # guess default soc-unit + if default_soc_unit is None: + if self.sensor.unit in ("MWh", "kWh"): + default_soc_unit = self.sensor.unit + elif self.sensor.unit in ("MW", "kW"): + default_soc_unit = self.sensor.unit + "h" + self.soc_maxima = VariableQuantityField( to_unit="MWh", default_src_unit=default_soc_unit,