diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index fe2d4118b..4a48e0cd5 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -4,6 +4,18 @@ Next release All changes ----------- +- :pull:`353`: Add meta functionality. + + - :meth:`.Platform.add_model_name` using :meth:`.Backend.add_model_name` + - :meth:`.Platform.add_scenario_name` using :meth:`.Backend.add_scenario_name` + - :meth:`.Platform.get_model_names` using :meth:`.Backend.get_model_names` + - :meth:`.Platform.get_scenario_names` using :meth:`.Backend.get_scenario_names` + - :meth:`.Platform.get_meta` using :meth:`.Backend.get_meta` + - :meth:`.Platform.set_meta` using :meth:`.Backend.set_meta` + - :meth:`.Platform.remove_meta` using :meth:`.Backend.remove_meta` + - :meth:`.Scenario.remove_meta` using :meth:`.Backend.remove_meta` + - deprecate :meth:`.Scenario.delete_meta` + - :pull:`349`: Avoid modifying indexers dictionary in :meth:`.AttrSeries.sel`. - :pull:`343`: Add region/unit parameters to :meth:`.Platform.export_timeseries_data`. - :pull:`347`: Preserve dtypes of index columns in :func:`.data_for_quantity`. diff --git a/doc/README.rst b/doc/README.rst index 7e101405a..244b18eed 100644 --- a/doc/README.rst +++ b/doc/README.rst @@ -28,7 +28,8 @@ There are a number of guides out there, e.g. on docutils_. Building the docs locally ------------------------- -Install the dependencies, above. +Install the dependencies, above. Repeat the installation steps to be able to +refer to code that was changed since the initial installation. From the command line, run:: diff --git a/doc/source/api-backend.rst b/doc/source/api-backend.rst index ebb31c043..855d49eb4 100644 --- a/doc/source/api-backend.rst +++ b/doc/source/api-backend.rst @@ -93,17 +93,24 @@ Backend API .. autosummary:: :nosignatures: + add_model_name + add_scenario_name close_db get_auth get_doc get_log_level + get_meta + get_model_names get_nodes get_scenarios + get_scenario_names get_units open_db read_file + remove_meta set_doc set_log_level + set_meta set_node set_unit write_file @@ -150,6 +157,7 @@ Backend API item_set_elements item_index list_items + remove_meta set_meta Methods related to :class:`message_ix.Scenario`: diff --git a/doc/source/api.rst b/doc/source/api.rst index c3bae648b..97620ddf4 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -29,11 +29,24 @@ Platform set_log_level units - The methods - :meth:`~.base.Backend.open_db`, - :meth:`~.base.Backend.close_db`, - :meth:`~.base.Backend.get_doc`, and - :meth:`~.base.Backend.set_doc` may also be called via Platform. + The following backend methods are available via Platform too: + + .. autosummary:: + backend.base.Backend.add_model_name + backend.base.Backend.add_scenario_name + backend.base.Backend.close_db + backend.base.Backend.get_doc + backend.base.Backend.get_meta + backend.base.Backend.get_model_names + backend.base.Backend.get_scenario_names + backend.base.Backend.open_db + backend.base.Backend.remove_meta + backend.base.Backend.set_doc + backend.base.Backend.set_meta + + These methods can be called like normal Platform methods, e.g.:: + + $ platform_instance.close_db() TimeSeries diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index e97e5a435..d8b1abf3f 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from copy import copy +from typing import Generator import json from ixmp.core import TimeSeries, Scenario @@ -21,6 +22,78 @@ def __call__(self, obj, method, *args, **kwargs): """ return getattr(self, method)(obj, *args, **kwargs) + # Platform methods + + def set_log_level(self, level): + """OPTIONAL: Set logging level for the backend and other code. + + The default implementation has no effect. + + Parameters + ---------- + level : int or Python logging level + + See also + -------- + get_log_level + """ + + def get_log_level(self): + """OPTIONAL: Get logging level for the backend and other code. + + The default implementation has no effect. + + Returns + ------- + str + Name of a :py:ref:`Python logging level `. + + See also + -------- + set_log_level + """ + + @abstractmethod + def set_doc(self, domain, docs): + """Save documentation to database + + Parameters + ---------- + domain : str + Documentation domain, e.g. model, scenario etc + docs : dict or array of tuples + Dictionary or tuple array containing mapping between name of domain + object (e.g. model name) and string representing fragment + of documentation + """ + + @abstractmethod + def get_doc(self, domain, name=None): + """ Read documentation from database + + Parameters + ---------- + domain : str + Documentation domain, e.g. model, scenario etc + name : str, optional + Name of domain entity (e.g. model name). + + Returns + ------- + str or dict + String representing fragment of documentation if name is passed as + parameter or dictionary containing mapping between name of domain + object (e.g. model name) and string representing fragment when + name parameter is omitted. + """ + + def open_db(self): + """OPTIONAL: (Re-)open database connection(s). + + A backend **may** connect to a database server. This method opens the + database connection if it is closed. + """ + def close_db(self): """OPTIONAL: Close database connection(s). @@ -53,6 +126,33 @@ def get_auth(self, user, models, kind): """ return {model: True for model in models} + @abstractmethod + def set_node(self, name, parent=None, hierarchy=None, synonym=None): + """Add a node name to the Platform. + + This method **must** have one of two effects, depending on the + arguments: + + - With `parent` and `hierarchy`: `name` is added as a child of `parent` + in the named `hierarchy`. + - With `synonym`: `synonym` is added as an alias for `name`. + + Parameters + ---------- + name : str + Node name. + parent : str, optional + Parent node name. + hierarchy : str, optional + Node hierarchy ID. + synonym : str, optional + Synonym for node. + + See also + -------- + get_nodes + """ + @abstractmethod def get_nodes(self): """Iterate over all nodes stored on the Platform. @@ -116,6 +216,46 @@ def set_timeslice(self, name, category, duration): get_timeslices """ + @abstractmethod + def add_model_name(self, name: str): + """Add (register) new model name. + + Parameters + ---------- + name : str + New model name + """ + + @abstractmethod + def add_scenario_name(self, name: str): + """Add (register) new scenario name. + + Parameters + ---------- + name : str + New scenario name + """ + + @abstractmethod + def get_model_names(self) -> Generator[str, None, None]: + """List existing model names. + + Returns + ------- + list of str + List of the retrieved model names. + """ + + @abstractmethod + def get_scenario_names(self) -> Generator[str, None, None]: + """List existing scenario names. + + Returns + ------- + list of str + List of the retrieved scenario names. + """ + @abstractmethod def get_scenarios(self, default, model, scenario): """Iterate over TimeSeries stored on the Platform. @@ -156,129 +296,32 @@ def get_scenarios(self, default, model, scenario): """ @abstractmethod - def get_units(self): - """Return all registered symbols for units of measurement. - - Returns - ------- - list of str - - See also - -------- - set_unit - """ - - def open_db(self): - """OPTIONAL: (Re-)open database connection(s). - - A backend **may** connect to a database server. This method opens the - database connection if it is closed. - """ - - def set_log_level(self, level): - """OPTIONAL: Set logging level for the backend and other code. - - The default implementation has no effect. + def set_unit(self, name, comment): + """Add a unit of measurement to the Platform. Parameters ---------- - level : int or Python logging level - - See also - -------- - get_log_level - """ - - def get_log_level(self): - """OPTIONAL: Get logging level for the backend and other code. - - The default implementation has no effect. - - Returns - ------- - str - Name of a :py:ref:`Python logging level `. + name : str + Symbol of the unit. + comment : str + Description of the change or of the unit. See also -------- - set_log_level - """ - - @abstractmethod - def set_doc(self, domain, docs): - """Save documentation to database - - Parameters - ---------- - domain : str - Documentation domain, e.g. model, scenario etc - docs : dict or array of tuples - Dictionary or tuple array containing mapping between name of domain - object (e.g. model name) and string representing fragment - of documentation + get_units """ @abstractmethod - def get_doc(self, domain, name=None): - """ Read documentation from database - - Parameters - ---------- - domain : str - Documentation domain, e.g. model, scenario etc - name : str, optional - Name of domain entity (e.g. model name). + def get_units(self): + """Return all registered symbols for units of measurement. Returns ------- - str or dict - String representing fragment of documentation if name is passed as - parameter or dictionary containing mapping between name of domain - object (e.g. model name) and string representing fragment when - name parameter is omitted. - """ - - @abstractmethod - def set_node(self, name, parent=None, hierarchy=None, synonym=None): - """Add a node name to the Platform. - - This method **must** have one of two effects, depending on the - arguments: - - - With `parent` and `hierarchy`: `name` is added as a child of `parent` - in the named `hierarchy`. - - With `synonym`: `synonym` is added as an alias for `name`. - - Parameters - ---------- - name : str - Node name. - parent : str, optional - Parent node name. - hierarchy : str, optional - Node hierarchy ID. - synonym : str, optional - Synonym for node. - - See also - -------- - get_nodes - """ - - @abstractmethod - def set_unit(self, name, comment): - """Add a unit of measurement to the Platform. - - Parameters - ---------- - name : str - Symbol of the unit. - comment : str - Description of the change or of the unit. + list of str See also -------- - get_units + set_unit """ def read_file(self, path, item_type: ItemType, **kwargs): @@ -370,24 +413,6 @@ def write_file(self, path, item_type: ItemType, **kwargs): else: raise NotImplementedError - @staticmethod - def _handle_rw_filters(filters: dict): - """Helper for :meth:`read_file` and :meth:`write_file`. - - The `filters` argument is unpacked if the 'scenarios' key is a single - :class:`TimeSeries` object. A 2-tuple is returned of the object (or - :obj:`None`) and the remaining filters. - """ - ts = None - filters = copy(filters) - try: - if isinstance(filters['scenario'], TimeSeries): - ts = filters.pop('scenario') - except KeyError: - pass # Don't modify filters at all - - return ts, filters - # Methods for ixmp.TimeSeries @abstractmethod @@ -482,6 +507,82 @@ def commit(self, ts: TimeSeries, comment): None """ + @abstractmethod + def discard_changes(self, ts: TimeSeries): + """Discard changes to *ts* since the last :meth:`ts_check_out`. + + Returns + ------- + None + """ + + @abstractmethod + def set_as_default(self, ts: TimeSeries): + """Set the current :attr:`.TimeSeries.version` as the default. + + Returns + ------- + None + + See also + -------- + get + is_default + """ + + @abstractmethod + def is_default(self, ts: TimeSeries): + """Return :obj:`True` if *ts* is the default version. + + Returns + ------- + bool + + See also + -------- + get + set_as_default + """ + + @abstractmethod + def last_update(self, ts: TimeSeries): + """Return the date of the last modification of the *ts*. + + Returns + ------- + str or None + """ + + @abstractmethod + def run_id(self, ts: TimeSeries): + """Return the run ID for the *ts*. + + Returns + ------- + int + """ + + def preload(self, ts: TimeSeries): + """OPTIONAL: Load *ts* data into memory.""" + + @staticmethod + def _handle_rw_filters(filters: dict): + """Helper for :meth:`read_file` and :meth:`write_file`. + + The `filters` argument is unpacked if the 'scenarios' key is a single + :class:`TimeSeries` object. A 2-tuple is returned of the object (or + :obj:`None`) and the remaining filters. + """ + ts = None + filters = copy(filters) + try: + if isinstance(filters['scenario'], TimeSeries): + ts = filters.pop('scenario') + except KeyError: + pass # Don't modify filters at all + + return ts, filters + @abstractmethod def get_data(self, ts: TimeSeries, region, variable, unit, year): """Retrieve time-series data. @@ -624,64 +725,6 @@ def delete_geo(self, ts: TimeSeries, region, variable, subannual, years, None """ - @abstractmethod - def discard_changes(self, ts: TimeSeries): - """Discard changes to *ts* since the last :meth:`ts_check_out`. - - Returns - ------- - None - """ - - @abstractmethod - def set_as_default(self, ts: TimeSeries): - """Set the current :attr:`.TimeSeries.version` as the default. - - Returns - ------- - None - - See also - -------- - get - is_default - """ - - @abstractmethod - def is_default(self, ts: TimeSeries): - """Return :obj:`True` if *ts* is the default version. - - Returns - ------- - bool - - See also - -------- - get - set_as_default - """ - - @abstractmethod - def last_update(self, ts: TimeSeries): - """Return the date of the last modification of the *ts*. - - Returns - ------- - str or None - """ - - @abstractmethod - def run_id(self, ts: TimeSeries): - """Return the run ID for the *ts*. - - Returns - ------- - int - """ - - def preload(self, ts: TimeSeries): - """OPTIONAL: Load *ts* data into memory.""" - # Methods for ixmp.Scenario @abstractmethod @@ -888,31 +931,49 @@ def item_delete_elements(self, s: Scenario, type, name, keys): """ @abstractmethod - def get_meta(self, s: Scenario): - """Return all meta. + def get_meta(self, model: str, scenario: str, version: int, strict: bool + ) -> dict: + """Retrieve meta indicators. + + Parameters + ---------- + model : str, optional + filter meta by a model + scenario : str, optional + filter meta by a scenario + version : int or str, optional + retrieve meta of a specific model/scenario run version + strict : bool, optional + only retrieve indicators from the requested model-scenario-version + level Returns ------- dict (str -> any) - Mapping from meta keys to values. + Mapping from meta category keys to values. - See also - -------- - s_get_meta + Raises + ------ + ValueError + On unsupported model-scenario-version combinations. + Supported combinations are: (model), (scenario), (model, scenario), + (model, scenario, version) """ @abstractmethod - def set_meta(self, s: Scenario, name_or_dict, value=None): - """Set single or multiple meta entries. + def set_meta(self, meta: dict, model: str, scenario: str, version: int): + """Set meta categories. Parameters ---------- - name_or_dict : str or dict - If the argument is dict, it used as a mapping of meta - categories (names) to values. Otherwise, use the argument - as the meta attribute name. - value : str or number or bool, optional - Meta attribute value. + meta : dict + containing meta key/value category pairs + model : str, optional + model name that meta should be attached to + scenario : str, optional + scenario name that meta should be attached to + version : int, optional + run version that meta should be attached to Returns ------- @@ -920,22 +981,38 @@ def set_meta(self, s: Scenario, name_or_dict, value=None): Raises ------ - TypeError - If *value* is not a valid type. + ValueError + On unsupported model-scenario-version combinations. + Supported combinations are: (model), (scenario), (model, scenario), + (model, scenario, version) """ @abstractmethod - def delete_meta(self, s, name): - """Remove single or multiple meta entries. + def remove_meta(self, categories: list, model: str, scenario: str, + version: int): + """Remove meta categories. Parameters ---------- - name : str or list of str - Either single meta key or list of keys. + categories : list of str + meta-category keys to remove + model : str, optional + only remove meta of a specific model + scenario : str, optional + only remove meta of a specific scenario + version : int, optional + only remove meta of a specific model/scenario run version Returns ------- None + + Raises + ------ + ValueError + On unsupported model-scenario-version combinations. + Supported combinations are: (model), (scenario), (model, scenario), + (model, scenario, version) """ @abstractmethod diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index c924cd230..8a57adb1b 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -1,6 +1,7 @@ from copy import copy from collections import ChainMap from collections.abc import Iterable, Sequence +from typing import Generator import gc from itertools import chain import logging @@ -19,7 +20,6 @@ from . import FIELDS, ItemType from .base import CachingBackend - log = logging.getLogger(__name__) @@ -53,12 +53,14 @@ 'java.lang.Integer', 'java.lang.NoClassDefFoundError', 'java.lang.IllegalArgumentException', + 'java.lang.Long', 'java.lang.Runtime', 'java.lang.System', 'java.math.BigDecimal', 'java.util.HashMap', 'java.util.LinkedHashMap', 'java.util.LinkedList', + 'java.util.ArrayList', 'java.util.Properties', 'at.ac.iiasa.ixmp.dto.DocumentationKey', ] @@ -142,6 +144,28 @@ def _domain_enum(domain): f'existing domains: {domains}') +def _unwrap(v): + """Unwrap meta numeric value or list of values (BigDecimal -> Double).""" + if isinstance(v, java.BigDecimal): + return v.doubleValue() + if isinstance(v, java.ArrayList): + return [_unwrap(elt) for elt in v] + return v + + +def _wrap(value): + if isinstance(value, (str, bool)): + return value + if isinstance(value, (int, float)): + return java.BigDecimal(value) + elif isinstance(value, (Sequence, Iterable)): + jlist = java.ArrayList() + jlist.addAll([_wrap(elt) for elt in value]) + return jlist + else: + raise ValueError(f'Cannot use value {value} as metadata') + + class JDBCBackend(CachingBackend): """Backend using JPype/JDBC to connect to Oracle and HyperSQL databases. @@ -251,6 +275,18 @@ def __init__(self, jvmargs=None, **kwargs): # Set the log level self.set_log_level(log_level) + def __del__(self): + self.close_db() + + @classmethod + def gc(cls): + if _GC_AGGRESSIVE: + # log.debug('Collect garbage') + java.System.gc() + gc.collect() + # else: + # log.debug('Skip garbage collection') + # Platform methods def set_log_level(self, level): @@ -331,6 +367,20 @@ def get_timeslices(self): def set_timeslice(self, name, category, duration): self.jobj.addTimeslice(name, category, java.Double(duration)) + def add_model_name(self, name): + self.jobj.addModel(str(name)) + + def add_scenario_name(self, name): + self.jobj.addScenario(str(name)) + + def get_model_names(self) -> Generator[str, None, None]: + for model in self.jobj.listModels(): + yield str(model) + + def get_scenario_names(self) -> Generator[str, None, None]: + for scenario in self.jobj.listScenarios(): + yield str(scenario) + def get_scenarios(self, default, model, scenario): # List> scenarios = self.jobj.getScenarioList(default, model, scenario) @@ -503,6 +553,22 @@ def _index_and_set_attrs(self, jobj, ts): ts.scheme = s + def _validate_meta_args(self, model, scenario, version): + """Validate arguments for getting/setting/deleting meta""" + valid = False + if model and not scenario and version is None: + valid = True + elif scenario and not model and version is None: + valid = True + elif model and scenario and version is None: + valid = True + elif model and scenario and version is not None: + valid = True + if not valid: + msg = ('Invalid arguments. Valid combinations are: (model), ' + '(scenario), (model, scenario), (model, scenario, version)') + raise ValueError(msg) + def init(self, ts, annotation): klass = ts.__class__.__name__ @@ -667,6 +733,7 @@ def delete_geo(self, ts, region, variable, subannual, years, unit): self.jindex[ts].removeGeoData(region, variable, subannual, years, unit) # Scenario methods + def clone(self, s, platform_dest, model, scenario, annotation, keep_solution, first_model_year=None): # Raise exceptions for limitations of JDBCBackend @@ -883,38 +950,33 @@ def item_delete_elements(self, s, type, name, keys): args = (s,) if type == 'set' else (s, type, name) self.cache_invalidate(*args) - def get_meta(self, s): - def unwrap(v): - """Unwrap meta numeric value (BigDecimal -> Double)""" - return v.doubleValue() if isinstance(v, java.BigDecimal) else v - - return {entry.getKey(): unwrap(entry.getValue()) - for entry in self.jindex[s].getMeta().entrySet()} - - def set_meta(self, s, name_or_dict, value=None): - if type(name_or_dict) == list: - jdata = java.LinkedHashMap() - for k, v in name_or_dict: - jdata.put(str(k), v) - self.jindex[s].setMeta(jdata) - return - - _type = type(value) - try: - _type = {int: 'Num', float: 'Num', str: 'Str', bool: 'Bool'}[_type] - method_name = 'setMeta' + _type - except KeyError: - raise TypeError(f'Cannot store meta of type {_type}') - - getattr(self.jindex[s], method_name)(name_or_dict, value) - - def delete_meta(self, s, name): - if type(name) == str: - name = [name] - jdata = java.LinkedList() - for k in name: - jdata.add(str(k)) - self.jindex[s].removeMeta(jdata) + def get_meta(self, model: str = None, scenario: str = None, + version: int = None, strict: bool = False) -> dict: + self._validate_meta_args(model, scenario, version) + if version is not None: + version = java.Long(version) + meta = self.jobj.getMeta(model, scenario, version, strict) + return {entry.getKey(): _unwrap(entry.getValue()) + for entry in meta.entrySet()} + + def set_meta(self, meta: dict, model: str = None, scenario: str = None, + version: int = None) -> None: + self._validate_meta_args(model, scenario, version) + if version is not None: + version = java.Long(version) + + jmeta = java.HashMap() + for k, v in meta.items(): + jmeta.put(str(k), _wrap(v)) + self.jobj.setMeta(model, scenario, version, jmeta) + + def remove_meta(self, categories, model: str = None, scenario: str = None, + version: int = None): + self._validate_meta_args(model, scenario, version) + if version is not None: + version = java.Long(version) + return self.jobj.removeMeta(model, scenario, version, + to_jlist(categories)) def clear_solution(self, s, from_year=None): from ixmp.core import Scenario @@ -967,18 +1029,6 @@ def _get_item(self, s, ix_type, name, load=True): else: # pragma: no cover _raise_jexception(e) - def __del__(self): - self.close_db() - - @classmethod - def gc(cls): - if _GC_AGGRESSIVE: - # log.debug('Collect garbage') - java.System.gc() - gc.collect() - # else: - # log.debug('Skip garbage collection') - def start_jvm(jvmargs=None): """Start the Java Virtual Machine via :mod:`JPype`. diff --git a/ixmp/core.py b/ixmp/core.py index 6d78f987e..095e535bd 100644 --- a/ixmp/core.py +++ b/ixmp/core.py @@ -57,10 +57,17 @@ class Platform: # List of method names which are handled directly by the backend _backend_direct = [ - 'open_db', + 'add_model_name', + 'add_scenario_name', 'close_db', 'get_doc', + 'get_meta', + 'get_model_names', + 'get_scenario_names', + 'open_db', + 'remove_meta', 'set_doc', + 'set_meta', ] def __init__(self, name=None, backend=None, **backend_args): @@ -1584,9 +1591,10 @@ def get_meta(self, name=None): Parameters ---------- name : str, optional - meta attribute name + meta category name """ - all_meta = self._backend('get_meta') + all_meta = self.platform._backend.get_meta(self.model, self.scenario, + self.version) return all_meta[name] if name else all_meta def set_meta(self, name_or_dict, value=None): @@ -1597,23 +1605,45 @@ def set_meta(self, name_or_dict, value=None): name_or_dict : str or dict If the argument is dict, it used as a mapping of meta categories (names) to values. Otherwise, use the argument - as the meta attribute name. + as the meta category name. value : str or number or bool, optional - Meta attribute value. + Meta category value. + """ + if not isinstance(name_or_dict, dict): + if isinstance(name_or_dict, str): + name_or_dict = {name_or_dict: value} + else: + msg = ('Unsupported parameter type of name_or_dict: %s. ' + 'Supported parameter types for name_or_dict are ' + 'String and Dictionary') % type(name_or_dict) + raise ValueError(msg) + self.platform._backend.set_meta(name_or_dict, self.model, + self.scenario, self.version) + + def delete_meta(self, *args, **kwargs): + """DEPRECATED: Remove scenario meta. + + Parameters + ---------- + name : str or list of str + Either single meta key or list of keys. """ - if type(name_or_dict) == dict: - name_or_dict = list(name_or_dict.items()) - self._backend('set_meta', name_or_dict, value) + warn('Scenario.delete_meta is deprecated; use Scenario.remove_meta ' + 'instead', DeprecationWarning) + self.remove_meta(*args, **kwargs) - def delete_meta(self, name): - """Delete scenario meta. + def remove_meta(self, name): + """Remove scenario meta. Parameters ---------- name : str or list of str Either single meta key or list of keys. """ - self._backend('delete_meta', name) + if isinstance(name, str): + name = [name] + self.platform._backend.remove_meta(name, self.model, self.scenario, + self.version) # Input and output def to_excel(self, path, items=ItemType.SET | ItemType.PAR, max_row=None): diff --git a/ixmp/ixmp.jar b/ixmp/ixmp.jar index e79a4803d..a35c8e0c4 100644 Binary files a/ixmp/ixmp.jar and b/ixmp/ixmp.jar differ diff --git a/ixmp/tests/backend/test_base.py b/ixmp/tests/backend/test_base.py index 58e7ebed1..ebc484657 100644 --- a/ixmp/tests/backend/test_base.py +++ b/ixmp/tests/backend/test_base.py @@ -21,6 +21,8 @@ def noop(self, *args, **kwargs): pass class BE2(Backend): + add_model_name = noop + add_scenario_name = noop cat_get_elements = noop cat_list = noop cat_set_elements = noop @@ -38,8 +40,10 @@ class BE2(Backend): get_doc = noop get_geo = noop get_meta = noop + get_model_names = noop get_nodes = noop get_scenarios = noop + get_scenario_names = noop get_timeslices = noop get_units = noop has_solution = noop @@ -52,12 +56,13 @@ class BE2(Backend): item_set_elements = noop last_update = noop list_items = noop + remove_meta = noop run_id = noop set_as_default = noop set_data = noop + set_doc = noop set_geo = noop set_meta = noop - set_doc = noop set_node = noop set_timeslice = noop set_unit = noop diff --git a/ixmp/tests/core/test_meta.py b/ixmp/tests/core/test_meta.py new file mode 100644 index 000000000..9f7d03628 --- /dev/null +++ b/ixmp/tests/core/test_meta.py @@ -0,0 +1,314 @@ +"""Test meta functionality of ixmp.Platform and ixmp.Scenario.""" + +import copy +import pytest + +import ixmp +from ixmp.testing import models + + +SAMPLE_META = {'sample_int': 3, 'sample_string': 'string_value', + 'sample_bool': False} +META_ENTRIES = [ + {'sample_int': 3}, + {'sample_string': 'string_value'}, + {'sample_bool': False}, + { + 'sample_int': 3, + 'sample_string': 'string_value', + 'sample_bool': False, + }, + {'mixed_category': ['string', 0.01, 2, True]}, +] +DANTZIG = models['dantzig'] + + +@pytest.mark.parametrize('meta', META_ENTRIES) +def test_set_meta_missing_argument(mp, meta): + with pytest.raises(ValueError): + mp.set_meta(meta) + with pytest.raises(ValueError): + mp.set_meta(meta, model=DANTZIG['model'], version=0) + with pytest.raises(ValueError): + mp.set_meta(meta, scenario=DANTZIG['scenario'], version=0) + + +@pytest.mark.parametrize('meta', META_ENTRIES) +def test_set_get_meta(mp, meta): + """Assert that storing+retrieving meta yields expected values.""" + mp.set_meta(meta, model=DANTZIG['model']) + obs = mp.get_meta(model=DANTZIG['model']) + assert obs == meta + + +@pytest.mark.parametrize('meta', META_ENTRIES) +def test_unique_meta(mp, meta): + """ + When setting a meta category on two distinct levels, a uniqueness error is + expected. + """ + scenario = ixmp.Scenario(mp, **DANTZIG, version='new') + scenario.commit('save dummy scenario') + mp.set_meta(meta, model=DANTZIG['model']) + expected = (r"The meta category .* is already used at another level: " + r"model canning problem, scenario null, version null") + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG, version=scenario.version) + scen = ixmp.Scenario(mp, **DANTZIG) + with pytest.raises(Exception, match=expected): + scen.set_meta(meta) + # changing the category value type of an entry should also raise an error + meta = {'sample_entry': 3} + mp.set_meta(meta, **DANTZIG) + meta['sample_entry'] = 'test-string' + expected = (r"The meta category .* is already used at another level: " + r"model canning problem, scenario standard, version null") + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG, version=scenario.version) + + +@pytest.mark.parametrize('meta', META_ENTRIES) +def test_set_get_meta_equals(mp, meta): + initial_meta = mp.get_meta(scenario=DANTZIG['scenario']) + mp.set_meta(meta, model=DANTZIG['model']) + obs_meta = mp.get_meta(scenario=DANTZIG['scenario']) + assert obs_meta == initial_meta + + +@pytest.mark.parametrize('meta', META_ENTRIES) +def test_unique_meta_model_scenario(mp, meta): + """ + When setting a meta key for a Model, it shouldn't be possible to set it + for a Model+Scenario then. + """ + mp.set_meta(meta, model=DANTZIG['model']) + expected = r"The meta category .* is already used at another level: " + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG) + + # Setting this meta category on a new model should fail too + dantzig2 = { + 'model': 'canning problem 2', + 'scenario': 'standard', + } + mp.add_model_name(dantzig2['model']) + expected = r"The meta category .* is already used at another level: " + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **dantzig2) + + +@pytest.mark.parametrize('meta', META_ENTRIES) +def test_get_meta_strict(mp, meta): + """ + Set meta indicators on several model/scenario/version levels and test + the 'strict' parameter of get_meta(). + """ + # set meta on various levels + model_meta = {'model_int': 3, 'model_string': 'string_value', + 'model_bool': False} + scenario_meta = {'scenario_int': 3, 'scenario_string': 'string_value', + 'scenario_bool': False} + meta2 = {'sample_int2': 3, 'sample_string2': 'string_value2', + 'sample_bool2': False} + meta3 = {'sample_int3': 3, 'sample_string3': 'string_value3', + 'sample_bool3': False, 'mixed3': ['string', 0.01, 2, True]} + meta_scen = {'sample_int4': 3, 'sample_string4': 'string_value4', + 'sample_bool4': False, 'mixed4': ['string', 0.01, 2, True]} + scenario2 = 'standard 2' + model2 = 'canning problem 2' + mp.add_scenario_name(scenario2) + mp.add_model_name(model2) + dantzig2 = { + 'model': model2, + 'scenario': 'standard', + } + dantzig3 = { + 'model': model2, + 'scenario': scenario2, + } + mp.set_meta(model_meta, model=DANTZIG['model']) + mp.set_meta(scenario_meta, scenario=DANTZIG['scenario']) + mp.set_meta(meta, **DANTZIG) + mp.set_meta(meta2, **dantzig2) + mp.set_meta(meta3, **dantzig3) + scen = ixmp.Scenario(mp, **DANTZIG, version="new") + scen.commit('save dummy scenario') + scen.set_meta(meta_scen) + + # Retrieve and validate meta indicators + # model + obs1 = mp.get_meta(model=DANTZIG['model']) + assert obs1 == model_meta + # scenario + obs2 = mp.get_meta(scenario=DANTZIG['scenario'], strict=True) + assert obs2 == scenario_meta + # model+scenario + obs3 = mp.get_meta(**DANTZIG) + exp3 = copy.copy(meta) + exp3.update(model_meta) + exp3.update(scenario_meta) + assert obs3 == exp3 + # model+scenario, strict + obs3_strict = mp.get_meta(**DANTZIG, strict=True) + assert obs3_strict == meta + assert obs3 != obs3_strict + + # second model+scenario combination + obs4 = mp.get_meta(**dantzig2) + exp4 = copy.copy(meta2) + exp4.update(scenario_meta) + assert obs4 == exp4 + # second model+scenario combination, strict + obs4_strict = mp.get_meta(**dantzig2, strict=True) + assert obs4_strict == meta2 + assert obs4 != obs4_strict + + # second model+scenario combination + obs5 = mp.get_meta(**dantzig3) + exp5 = copy.copy(meta3) + assert obs5 == exp5 + + # model+scenario+version + obs6 = mp.get_meta(**DANTZIG, version=scen.version) + exp6 = copy.copy(meta_scen) + exp6.update(meta) + exp6.update(model_meta) + exp6.update(scenario_meta) + assert obs6 == exp6 + obs6_strict = mp.get_meta(DANTZIG['model'], DANTZIG['scenario'], + scen.version, strict=True) + assert obs6_strict == meta_scen + + +@pytest.mark.parametrize('meta', META_ENTRIES) +def test_unique_meta_scenario(mp, meta): + """ + When setting a meta key on a specific Scenario run, setting the same key + on an higher level (Model or Model+Scenario) should fail. + """ + scen = ixmp.Scenario(mp, **DANTZIG) + scen.set_meta(meta) + # add a second scenario and verify that setting+getting Meta works + scen2 = ixmp.Scenario(mp, **DANTZIG, version="new") + scen2.commit('save dummy scenario') + scen2.set_meta(meta) + assert scen2.get_meta() == scen.get_meta() + + expected = ("Meta indicators already contain category") + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, **DANTZIG) + with pytest.raises(Exception, match=expected): + mp.set_meta(meta, model=DANTZIG['model']) + + +def test_meta_partial_overwrite(mp): + meta1 = {'sample_string': 3.0, 'another_string': 'string_value', + 'sample_bool': False} + meta2 = {'sample_string': 5.0, 'yet_another_string': 'hello', + 'sample_bool': True} + scen = ixmp.Scenario(mp, **DANTZIG) + scen.set_meta(meta1) + scen.set_meta(meta2) + expected = copy.copy(meta1) + expected.update(meta2) + obs = scen.get_meta() + assert obs == expected + + +def test_remove_meta(mp): + meta = {'sample_int': 3.0, 'another_string': 'string_value'} + remove_key = 'another_string' + mp.set_meta(meta, **DANTZIG) + mp.remove_meta(remove_key, **DANTZIG) + expected = copy.copy(meta) + del expected[remove_key] + obs = mp.get_meta(**DANTZIG) + assert expected == obs + + +def test_remove_invalid_meta(mp): + """ + Removing nonexisting meta entries or None shouldn't result in any meta + being removed. Providing None should give a ValueError. + """ + mp.set_meta(SAMPLE_META, **DANTZIG) + with pytest.raises(ValueError): + mp.remove_meta(None, **DANTZIG) + mp.remove_meta('nonexisting_category', **DANTZIG) + mp.remove_meta([], **DANTZIG) + obs = mp.get_meta(**DANTZIG) + assert obs == SAMPLE_META + + +def test_set_and_remove_meta_scenario(mp): + """ + Test partial overwriting and meta deletion on scenario level. + """ + meta1 = {'sample_string': 3.0, 'another_string': 'string_value'} + meta2 = {'sample_string': 5.0, 'yet_another_string': 'hello'} + remove_key = 'another_string' + + scen = ixmp.Scenario(mp, **DANTZIG) + scen.set_meta(meta1) + scen.set_meta(meta2) + expected = copy.copy(meta1) + expected.update(meta2) + obs = scen.get_meta() + assert expected == obs + + scen.remove_meta(remove_key) + del expected[remove_key] + obs = scen.get_meta() + assert obs == expected + + +def test_scenario_delete_meta_warning(mp): + """ + Scenario.delete_meta works but raises a deprecation warning. + + This test can be removed once Scenario.delete_meta is removed. + """ + scen = ixmp.Scenario(mp, **DANTZIG) + meta = {'sample_int': 3, 'sample_string': 'string_value'} + remove_key = 'sample_string' + + scen.set_meta(meta) + with pytest.warns(DeprecationWarning): + scen.delete_meta(remove_key) + expected = copy.copy(meta) + del expected[remove_key] + obs = scen.get_meta() + assert obs == expected + + +def test_meta_arguments(mp): + """Set scenario meta with key-value arguments""" + meta = {'sample_int': 3} + scen = ixmp.Scenario(mp, **DANTZIG) + scen.set_meta(meta) + # add a second scenario and verify that setting Meta for it works + scen2 = ixmp.Scenario(mp, **DANTZIG, version="new") + scen2.commit('save dummy scenario') + scen2.set_meta(*meta.popitem()) + assert scen.get_meta() == scen2.get_meta() + + +def test_update_meta_lists(mp): + """Set metadata categories having list/array values.""" + SAMPLE_META = {'list_category': ['a', 'b', 'c']} + mp.set_meta(SAMPLE_META, model=DANTZIG['model']) + obs = mp.get_meta(model=DANTZIG['model']) + assert obs == SAMPLE_META + # try updating meta + SAMPLE_META = {'list_category': ['a', 'e', 'f']} + mp.set_meta(SAMPLE_META, model=DANTZIG['model']) + obs = mp.get_meta(model=DANTZIG['model']) + assert obs == SAMPLE_META + + +def test_meta_mixed_list(mp): + """Set metadata categories having list/array values.""" + meta = {'mixed_category': ['string', 0.01, True]} + mp.set_meta(meta, model=DANTZIG['model']) + obs = mp.get_meta(model=DANTZIG['model']) + assert obs == meta diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index 11671076e..a7219b995 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -1,4 +1,4 @@ -"""Test all functionality of ixmp.Platform.""" +"""Test functionality of ixmp.Platform.""" from sys import getrefcount from weakref import getweakrefcount @@ -192,3 +192,13 @@ def test_weakref(): assert s.model == 'foo' # *s* is garbage-collected at this point + + +def test_add_model_name(mp): + mp.add_model_name('new_model_name') + assert 'new_model_name' in mp.get_model_names() + + +def test_add_scenario_name(mp): + mp.add_scenario_name('new_scenario_name') + assert 'new_scenario_name' in mp.get_scenario_names() diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index 4b9cfa76a..3dd03a338 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -310,7 +310,7 @@ def test_meta(self, mp, test_dict): 'test_number_negative', 'test_bool_false'} # Setting with a type other than int, float, bool, str raises TypeError - with pytest.raises(TypeError, match='Cannot store meta of type'): + with pytest.raises(ValueError, match='Cannot use value'): scen.set_meta('test_string', complex(1, 1)) def test_meta_bulk(self, mp, test_dict):