diff --git a/.appveyor.yml b/.appveyor.yml index 2a30886d0..f55cc6cf4 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,6 +4,8 @@ environment: # Use the JDK installed on AppVeyor images JAVA_HOME: C:\Program Files\Java\jdk13 + # Always display verbose exceptions in JDBCBackend + IXMP_JDBC_EXCEPTION_VERBOSE: '1' matrix: - PYTHON_VERSION: "3.6" - PYTHON_VERSION: "3.7" diff --git a/.travis.yml b/.travis.yml index 60ca4b195..83b16815e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ # Continuous integration configuration for Travis +# NB use https://config.travis-ci.com/explore to validate changes dist: xenial @@ -6,15 +7,17 @@ language: r r: release -matrix: - include: - - os: linux - env: PYENV=py37 - - os: osx - env: PYENV=py37 -# turn these on once travis support gets a little better, see pyam for example -# - os: windows -# env: PYENV=py37 +# - Build against Python 3.7 +# - Always display verbose exceptions in JDBCBackend +env: + PYENV=py37 + IXMP_JDBC_EXCEPTION_VERBOSE=1 + +os: +- linux +- osx +# TODO turn this on once Travis Windows support improves +# - windows r_packages: - IRkernel diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 197307252..17241a2a0 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -4,6 +4,7 @@ Next release All changes ----------- +- `#286 `_: Add :meth:`.Scenario.to_excel` and :meth:`.read_excel`; this functionality is transferred to ixmp from :mod:`message_ix`. - `#270 `_: Include all tests in the ixmp package. - `#212 `_: Add :meth:`Model.initialize` API to help populate new Scenarios according to a model scheme. - `#267 `_: Apply units to reported quantities. diff --git a/ci/conda-requirements.txt b/ci/conda-requirements.txt index 3c7c6f87a..6de16f540 100644 --- a/ci/conda-requirements.txt +++ b/ci/conda-requirements.txt @@ -8,7 +8,7 @@ numpydoc pandas pint pytest-cov -pytest>=3.9 +pytest>=5 PyYAML sparse sphinx diff --git a/doc/source/api-backend.rst b/doc/source/api-backend.rst index ba02d8780..af0f45e98 100644 --- a/doc/source/api-backend.rst +++ b/doc/source/api-backend.rst @@ -36,6 +36,10 @@ Provided backends .. tip:: Modifying an item by adding or deleting elements invalidates its cache. + JDBCBackend has the following limitations: + + - The `comment` argument to :meth:`Platform.add_unit` is limited to 64 characters. + .. automethod:: ixmp.backend.jdbc.start_jvm Backend API @@ -86,6 +90,7 @@ Backend API close_db get_auth + get_log_level get_nodes get_scenarios get_units diff --git a/doc/source/api-python.rst b/doc/source/api-python.rst index 5c6b9d300..c3d252576 100644 --- a/doc/source/api-python.rst +++ b/doc/source/api-python.rst @@ -72,6 +72,7 @@ TimeSeries is_default last_update preload_timeseries + read_file remove_geodata remove_timeseries run_id @@ -144,6 +145,7 @@ Scenario load_scenario_data par par_list + read_excel remove_par remove_set remove_solution @@ -152,6 +154,7 @@ Scenario set_list set_meta solve + to_excel var var_list diff --git a/doc/source/api.rst b/doc/source/api.rst index a5c8b50fd..b90bfc423 100644 --- a/doc/source/api.rst +++ b/doc/source/api.rst @@ -10,6 +10,7 @@ On separate pages: api-python api-backend + file-io api-model reporting diff --git a/doc/source/file-io.rst b/doc/source/file-io.rst new file mode 100644 index 000000000..40decd238 --- /dev/null +++ b/doc/source/file-io.rst @@ -0,0 +1,78 @@ +File formats and input/output +***************************** + +In addition to the data management features provided by :doc:`api-backend`, ixmp is able to write and read :class:`TimeSeries` and :class:`Scenario` data to and from files. +This page describes those options and formats. + +Time series data +================ + +Time series data can be: + +- Read using :meth:`.import_timeseries`, or the CLI command ``ixmp import timeseries FILE`` for a single TimeSeries object. +- Written using :meth:`.export_timeseries_data` for multiple TimeSeries objects at once. + +Both CSV and Excel files in the IAMC time-series format are supported. + +.. _excel-data-format: + +Scenario/model data +=================== + +Scenario data can be read from/written to Microsoft Excel files using :meth:`.Scenario.read_excel` and :meth:`.to_excel`, and the CLI commands ``ixmp import scenario FILE`` and ``ixmp export FILE``. +The files have the following structure: + +- One sheet named 'ix_type_mapping' with two columns: + + - 'item': the name of an ixmp item. + - 'ix_type': the item's type as a length-3 string: 'set', 'par', 'var', or 'equ'. + +- One sheet per item. +- Sets: + + - Sheets for index sets have one column, with a header cell that is the set name. + - Sheets for one-dimensional indexed sets have one column, with a header cell that is the index set name. + - Sheets for multi-dimensional indexed sets have multiple columns. + - Sets with no elements are represented by empty sheets. + +- Parameters, variables, and equations: + + - Sheets have zero (for scalar items) or more columns with headers that are the index *names* (not necessarily sets; see below) for those dimensions. + - Parameter sheets have 'value' and 'unit' columns. + - Variable and equation sheets have 'lvl' and 'mrg' columns. + - Items with no elements are not included in the file. + +Limitations +----------- + +Reading variables and equations + The ixmp API provides no way to set the data of variables and equations, because these are considered model solution data. + + Thus, while :meth:`.to_excel` will write files containing variable and equation data, :meth:`.read_excel` can not add these to a Scenario, and only emits log messages indicating that they are ignored. + +Multiple dimensions indexed by the same set + :meth:`.read_excel` provides the `init_items` argument to create new sets and parameters when reading a file. + However, the file format does not capture information needed to reconstruct the original data in all cases. + + For example:: + + scenario.init_set('foo') + scenario.add_set('foo', ['a', 'b', 'c']) + scenario.init_par(name='bar', idx_sets=['foo']) + scenario.init_par( + name='baz', + idx_sets=['foo', 'foo'], + idx_names=['foo', 'another_dimension']) + scenario.to_excel('file.xlsx') + + :file:`file.xlsx` will contain sheets named 'bar' and 'baz'. + The sheet 'bar' will have column headers 'foo', 'value', and 'unit', which are adequate to reconstruct the parameter. + However, the sheet 'baz' will have column headers 'foo' and 'another_dimension'; this information does not allow ixmp to infer that 'another_dimension' is indexed by 'foo'. + + To work around this limitation, initialize 'baz' with the correct dimensions before reading its data:: + + new_scenario.init_par( + name='baz', + idx_sets=['foo', 'foo'], + idx_names=['foo', 'another_dimension']) + new_scenario.read_excel('file.xlsx', init_items=True) diff --git a/ixmp/backend/base.py b/ixmp/backend/base.py index a62770dda..c15cb475e 100644 --- a/ixmp/backend/base.py +++ b/ixmp/backend/base.py @@ -4,6 +4,7 @@ from ixmp.core import TimeSeries, Scenario from . import ItemType +from .io import ts_read_file, s_read_excel, s_write_excel class Backend(ABC): @@ -177,9 +178,30 @@ def open_db(self): def set_log_level(self, level): """OPTIONAL: Set logging level for the backend and other code. + The default implementation has no effect. + Parameters ---------- level : int or Python logging level + + See also + -------- + get_log_level + """ + + def get_log_level(self): + """OPTIONAL: Get logging level for the backend and other code. + + The default implementation has no effect. + + Returns + ------- + str + Name of a :py:ref:`Python logging level `. + + See also + -------- + set_log_level """ @abstractmethod @@ -232,6 +254,12 @@ def read_file(self, path, item_type: ItemType, **kwargs): the `path` and `item_type` methods. For all other combinations, it **must** raise :class:`NotImplementedError`. + The default implementation supports: + + - `path` ending in '.xlsx', `item_type` is ItemType.MODEL: read a + single Scenario given by kwargs['filters']['scenario'] from file + using :meth:`pandas.DataFrame.read_excel`. + Parameters ---------- path : os.PathLike @@ -260,8 +288,13 @@ def read_file(self, path, item_type: ItemType, **kwargs): -------- write_file """ - # TODO move message_ix.core.read_excel here - raise NotImplementedError + s, filters = self._handle_rw_filters(kwargs.pop('filters', {})) + if path.suffix in ('.csv', '.xlsx') and item_type is ItemType.TS and s: + ts_read_file(s, path, **kwargs) + elif path.suffix == '.xlsx' and item_type is ItemType.MODEL and s: + s_read_excel(self, s, path, **kwargs) + else: + raise NotImplementedError def write_file(self, path, item_type: ItemType, **kwargs): """OPTIONAL: Write Platform, TimeSeries, or Scenario data to file. @@ -270,6 +303,12 @@ def write_file(self, path, item_type: ItemType, **kwargs): the `path` and `item_type` methods. For all other combinations, it **must** raise :class:`NotImplementedError`. + The default implementation supports: + + - `path` ending in '.xlsx', `item_type` is ItemType.MODEL: write a + single Scenario given by kwargs['filters']['scenario'] to file using + :meth:`pandas.DataFrame.to_excel`. + Parameters ---------- path : os.PathLike @@ -289,8 +328,11 @@ def write_file(self, path, item_type: ItemType, **kwargs): -------- read_file """ - # TODO move message_ix.core.to_excel here - raise NotImplementedError + s, filters = self._handle_rw_filters(kwargs.pop('filters', {})) + if path.suffix == '.xlsx' and item_type is ItemType.MODEL and s: + s_write_excel(self, s, path) + else: + raise NotImplementedError @staticmethod def _handle_rw_filters(filters: dict): @@ -355,6 +397,12 @@ def get(self, ts: TimeSeries, version): ------- None + Raises + ------ + ValueError + If :attr:`~.TimeSeries.model` or :attr:`~.TimeSeries.scenario` does + not exist on the Platform. + See also -------- ts_set_as_default @@ -742,7 +790,7 @@ def item_get_elements(self, s: Scenario, type, name, filters=None): When *type* is 'set' and *name* an index set (not indexed by other sets). dict - When *type* is 'equ', 'par', or 'set' and *name* is scalar (zero- + When *type* is 'equ', 'par', or 'var' and *name* is scalar (zero- dimensional). The value has the keys 'value' and 'unit' (for 'par') or 'lvl' and 'mrg' (for 'equ' or 'var'). pandas.DataFrame @@ -750,6 +798,11 @@ def item_get_elements(self, s: Scenario, type, name, filters=None): one column per index name with dimension values; plus the columns 'value' and 'unit' (for 'par') or 'lvl' and 'mrg' (for 'equ' or 'var'). + + Raises + ------ + KeyError + If *name* does not exist in *s*. """ @abstractmethod diff --git a/ixmp/backend/io.py b/ixmp/backend/io.py new file mode 100644 index 000000000..45edd72d1 --- /dev/null +++ b/ixmp/backend/io.py @@ -0,0 +1,245 @@ +import logging + +import pandas as pd + +from ixmp.utils import as_str_list + + +log = logging.getLogger(__name__) + + +def ts_read_file(ts, path, firstyear=None, lastyear=None): + """Read data from a CSV or Microsoft Excel file at *path* into *ts*. + + See also + -------- + TimeSeries.add_timeseries + TimeSeries.read_file + """ + + if path.suffix == '.csv': + df = pd.read_csv(path) + elif path.suffix == '.xlsx': + df = pd.read_excel(path) + + ts.check_out(timeseries_only=True) + ts.add_timeseries(df, year_lim=(firstyear, lastyear)) + + msg = f'adding timeseries data from {path}' + if firstyear: + msg += f' from {firstyear}' + if lastyear: + msg += f' until {lastyear}' + ts.commit(msg) + + +def s_write_excel(be, s, path): + """Write *s* to a Microsoft Excel file at *path*. + + See also + -------- + Scenario.to_excel + """ + # item name -> ixmp type + name_type = {} + for ix_type in ('set', 'par', 'var', 'equ'): + name_type.update({n: ix_type for n in be.list_items(s, ix_type)}) + + # Open file + writer = pd.ExcelWriter(path, engine='xlsxwriter') + + omitted = set() + + for name, ix_type in name_type.items(): + # Extract data: dict, pd.Series, or pd.DataFrame + data = be.item_get_elements(s, ix_type, name) + + if isinstance(data, dict): + # Scalar equ/par/var: series with index like 'value', 'unit'. + # Convert to DataFrame with 1 row. + data = pd.Series(data, name=name) \ + .to_frame() \ + .transpose() + elif isinstance(data, pd.Series): + # Index set: use own name as the header + data.name = name + + # Write empty sets, but not equ/par/var + if ix_type != 'set' and data.empty: + omitted.add(name) + continue + + data.to_excel(writer, sheet_name=name, index=False) + + # Discard entries that were not written + for name in omitted: + name_type.pop(name) + + # Write the name -> type map + pd.Series(name_type, name='ix_type') \ + .rename_axis(index='item') \ + .reset_index() \ + .to_excel(writer, sheet_name='ix_type_mapping', index=False) + + writer.save() + + +def maybe_init_item(scenario, ix_type, name, new_idx, path): + """Call :meth:`~.init_set`, :meth:`.init_par`, etc. if possible. + + Logs an intelligible warning and then raises ValueError: + + - the *new_idx* is ambiguous, containing names that cannot be used to infer + sets, or + - the init_*() call fails because of an existing item with index names + that are different from *new_idx*. + + """ + # Check for ambiguous index names + ambiguous_idx = sorted(set(new_idx or []) - set(scenario.set_list())) + if len(ambiguous_idx): + msg = (f'Cannot read {ix_type} {name!r}: index set(s) cannot be ' + f'inferred for name(s) {ambiguous_idx}') + log.warning(msg) + raise ValueError + + try: + # Initialize + getattr(scenario, f'init_{ix_type}')(name, new_idx) + except ValueError as e: + if 'exists' not in e.args[0]: # pragma: no cover + raise # Some other ValueError + + # Existing item; check that is has the same index names + + # [] and None are equivalent; convert to be consistent + existing = scenario.idx_names(name) or None + if isinstance(new_idx, list) and new_idx == []: + new_idx = None + + if existing != new_idx: + msg = (f'Existing {ix_type} {name!r} has index names(s) {existing}' + f' != {new_idx} in {path.name}') + log.warning(msg) + raise ValueError + + +def s_read_excel(be, s, path, add_units=False, init_items=False, + commit_steps=False): + """Read data from a Microsoft Excel file at *path* into *s*. + + See also + -------- + Scenario.read_excel + """ + log.info(f'Read data from {path}') + + # Get item name -> ixmp type mapping as a pd.Series + xf = pd.ExcelFile(path) + name_type = xf.parse('ix_type_mapping', index_col='item')['ix_type'] + + # List of *set name, data) to add + sets_to_add = [(n, None) for n in name_type.index[name_type == 'set']] + + # Add sets in two passes: + # 1. Index sets, required to initialize other sets. + # 2. Sets indexed by others. + for name, data in sets_to_add: + first_pass = data is None + if first_pass: + # Read data + data = xf.parse(name) + + if (first_pass and len(data.columns) == 1) or not first_pass: + # Index set or second appearance; add immediately + idx_sets = data.columns.to_list() + + if init_items: + # Determine index set(s) for this set + if len(idx_sets) == 1: + if idx_sets == [0]: # pragma: no cover + # Old-style export with uninformative '0' as a column + # header; assume it's an index set + log.warning(f"Add {name} with header '0' as index set") + idx_sets = None + elif idx_sets == [name]: + # Set's own name as column header -> an index set + idx_sets = None + else: + pass # 1-D set indexed by another set + + try: + maybe_init_item(s, 'set', name, idx_sets, path) + except ValueError: + continue # Ambiguous or conflicting; skip this set + + if len(data.columns) == 1: + # Convert data frame into 1-D vector + data = data.iloc[:, 0].values + + if idx_sets is not None and idx_sets != [name]: + # Indexed set must be input as list of list of str + data = list(map(as_str_list, data)) + + try: + s.add_set(name, data) + except KeyError: + raise ValueError(f'no set {name!r}; try init_items=True') + else: + # Reappend to the list to process later + sets_to_add.append((name, data)) + + if commit_steps: + s.commit(f'Loaded sets from {path}') + s.check_out() + + if add_units: + # List of existing units for reference + units = set(be.get_units()) + + # Add equ/par/var data + for name, ix_type in name_type[name_type != 'set'].items(): + if ix_type in ('equ', 'var'): + log.info(f'Cannot read {ix_type} {name!r}') + continue + + # Only parameters beyond this point + + df = xf.parse(name) + + if add_units: + # New units appearing in this parameter + to_add = set(df['unit'].unique()) - units + + for unit in to_add: + log.info(f'Add missing unit: {unit}') + # FIXME cannot use the comment f'Loaded from {path}' here; too + # long for JDBCBackend + be.set_unit(unit, f'Loaded from file') + + # Update the reference set to avoid re-adding these units + units |= to_add + + # NB if equ/var were imported, also need to filter 'lvl', 'mrg' here + idx_sets = list( + filter(lambda v: v not in ('value', 'unit'), df.columns) + ) + + if init_items: + try: + # Same as init_scalar if idx_sets == [] + maybe_init_item(s, ix_type, name, idx_sets, path) + except ValueError: + continue # Ambiguous or conflicting; skip this parameter + + if not len(idx_sets): + # No index sets -> scalar parameter; must supply empty 'key' column + # for add_par() + df['key'] = None + + s.add_par(name, df) + + if commit_steps: + # Commit after every parameter + s.commit(f'Loaded {ix_type} {name!r} from {path}') + s.check_out() diff --git a/ixmp/backend/jdbc.py b/ixmp/backend/jdbc.py index b123d1fde..0d0212a9f 100644 --- a/ixmp/backend/jdbc.py +++ b/ixmp/backend/jdbc.py @@ -21,14 +21,17 @@ log = logging.getLogger(__name__) +_EXCEPTION_VERBOSE = os.environ.get('IXMP_JDBC_EXCEPTION_VERBOSE', '0') == '1' + # Map of Python to Java log levels +# https://logging.apache.org/log4j/2.x/log4j-api/apidocs/org/apache/logging/log4j/Level.html LOG_LEVELS = { - 'CRITICAL': 'ALL', + 'CRITICAL': 'FATAL', 'ERROR': 'ERROR', 'WARNING': 'WARN', 'INFO': 'INFO', 'DEBUG': 'DEBUG', - 'NOTSET': 'OFF', + 'NOTSET': 'ALL', } # Java classes, loaded by start_jvm(). These become available as e.g. @@ -72,7 +75,7 @@ def _create_properties(driver=None, path=None, url=None, user=None, if url is None or path is not None: raise ValueError("use JDBCBackend(driver='oracle', url=…)") - full_url = 'jdbc:oracle:thin:@{}'.format(url) + full_url = f'jdbc:oracle:thin:@{url}' elif driver == 'hsqldb': if path is None and url is None: raise ValueError("use JDBCBackend(driver='hsqldb', path=…)") @@ -87,7 +90,7 @@ def _create_properties(driver=None, path=None, url=None, user=None, # URL spec url_path = (str(PurePosixPath(Path(path).resolve())) .replace('\\', '')) - full_url = 'jdbc:hsqldb:file:{}'.format(url_path) + full_url = f'jdbc:hsqldb:file:{url_path}' user = user or 'ixmp' password = password or 'ixmp' @@ -108,6 +111,15 @@ def _read_properties(file): return properties +def _raise_jexception(exc, msg='unhandled Java exception: '): + """Convert Java/JPype exceptions to ordinary Python RuntimeError.""" + if _EXCEPTION_VERBOSE: + msg += '\n\n' + exc.stacktrace() + else: + msg += exc.message() + raise RuntimeError(msg) from None + + class JDBCBackend(CachingBackend): """Backend using JPype/JDBC to connect to Oracle and HyperSQLDB instances. @@ -189,22 +201,18 @@ def __init__(self, jvmargs=None, **kwargs): try: self.jobj = java.Platform('Python', properties) except java.NoClassDefFoundError as e: # pragma: no cover - raise NameError( - '{}\nCheck that dependencies of ixmp.jar are included in {}' - .format(e, Path(__file__).parents[2] / 'lib')) + raise NameError(f'{e}\nCheck that dependencies of ixmp.jar are ' + f"included in {Path(__file__).parents[2] / 'lib'}") except jpype.JException as e: # pragma: no cover # Handle Java exceptions jclass = e.__class__.__name__ - info = '\n{}\n(Java: {})'.format(e, jclass) if jclass.endswith('HikariPool.PoolInitializationException'): redacted = copy(kwargs) redacted.update({'user': '(HIDDEN)', 'password': '(HIDDEN)'}) - raise RuntimeError('unable to connect to database:\n{!r}{}' - .format(redacted, info)) from None + msg = f'unable to connect to database:\n{redacted!r}' elif jclass.endswith('FlywayException'): - raise RuntimeError('when initializing database:' + info) - else: - raise RuntimeError('unhandled Java exception:' + info) from e + msg = f'when initializing database:' + _raise_jexception(e, f'{msg}\n(Java: {jclass})') # Invoke the parent constructor to initialize the cache super().__init__() @@ -214,6 +222,10 @@ def __init__(self, jvmargs=None, **kwargs): def set_log_level(self, level): self.jobj.setLogLevel(LOG_LEVELS[level]) + def get_log_level(self): + levels = {v: k for k, v in LOG_LEVELS.items()} + return levels.get(self.jobj.getLogLevel(), 'UNKNOWN') + def open_db(self): """(Re-)open the database connection.""" self.jobj.openDB() @@ -301,6 +313,14 @@ def read_file(self, path, item_type: ItemType, **kwargs): -------- .Backend.read_file """ + try: + # Call the default implementation, e.g. for .xlsx + super().read_file(path, item_type, **kwargs) + except NotImplementedError: + pass + else: + return + ts, filters = self._handle_rw_filters(kwargs.pop('filters', {})) if path.suffix == '.gdx' and item_type is ItemType.MODEL: kw = {'check_solution', 'comment', 'equ_list', 'var_list'} @@ -308,8 +328,8 @@ def read_file(self, path, item_type: ItemType, **kwargs): if not isinstance(ts, Scenario): raise ValueError('read from GDX requires a Scenario object') elif set(kwargs.keys()) != kw: - raise ValueError(('keyword arguments {} do not match required ' - '{}').format(kwargs.keys(), kw)) + raise ValueError(f'keyword arguments {kwargs.keys()} do not ' + f'match required {kw}') args = ( str(path.parent), @@ -321,7 +341,7 @@ def read_file(self, path, item_type: ItemType, **kwargs): ) if len(kwargs): - raise ValueError('extra keyword arguments {}'.format(kwargs)) + raise ValueError(f'extra keyword arguments {kwargs}') self.jindex[ts].readSolutionFromGDX(*args) @@ -353,6 +373,14 @@ def write_file(self, path, item_type: ItemType, **kwargs): -------- .Backend.write_file """ + try: + # Call the default implementation, e.g. for .xlsx + super().write_file(path, item_type, **kwargs) + except NotImplementedError: + pass + else: + return + ts, filters = self._handle_rw_filters(kwargs.pop('filters', {})) if path.suffix == '.gdx' and item_type is ItemType.SET | ItemType.PAR: if len(filters): @@ -405,7 +433,15 @@ def get(self, ts, version): try: jobj = method(*args) except java.IxException as e: - raise RuntimeError(*e.args) from None + # Try to re-raise as a ValueError for bad model or scenario name + match = re.search(r"table '([^']*)' from the database", e.args[0]) + if match: + param = match.group(1).lower() + if param in ('model', 'scenario'): + raise ValueError(f'{param}={getattr(ts, param)!r}') + + # Failed + _raise_jexception(e) # Add to index self.jindex[ts] = jobj @@ -427,10 +463,13 @@ def check_out(self, ts, timeseries_only): try: self.jindex[ts].checkOut(timeseries_only) except java.IxException as e: - raise RuntimeError(e) from None + _raise_jexception(e) def commit(self, ts, comment): - self.jindex[ts].commit(comment) + try: + self.jindex[ts].commit(comment) + except java.IxException as e: + _raise_jexception(e) if ts.version == 0: ts.version = self.jindex[ts].getVersion() @@ -554,12 +593,12 @@ def clone(self, s, platform_dest, model, scenario, annotation, f'Clone between {self.__class__} and' f'{platform_dest._backend.__class__}') elif platform_dest._backend is not self: - msg = 'Cross-platform clone of {}.Scenario with'.format( - s.__class__.__module__.split('.')[0]) + package = s.__class__.__module__.split('.')[0] + msg = f'Cross-platform clone of {package}.Scenario with' if keep_solution is False: - raise NotImplementedError(msg + ' `keep_solution=False`') + raise NotImplementedError(f'{msg} `keep_solution=False`') elif 'message_ix' in msg and first_model_year is not None: - raise NotImplementedError(msg + ' first_model_year != None') + raise NotImplementedError(f'{msg} first_model_year != None') # Prepare arguments args = [platform_dest._backend.jobj, model, scenario, annotation, @@ -603,13 +642,10 @@ def init_item(self, s, type, name, idx_sets, idx_names): try: func(name, idx_sets, idx_names) except jpype.JException as e: - e = str(e) - if 'This Scenario cannot be edited' in e: - raise RuntimeError(e) - elif 'already exists' in e: - raise ValueError('{!r} already exists'.format(name)) + if 'already exists' in e.args[0]: + raise ValueError(f'{name!r} already exists') else: - raise + _raise_jexception(e) def delete_item(self, s, type, name): getattr(self.jindex[s], f'remove{type.title()}')(name) @@ -747,7 +783,7 @@ def item_set_elements(self, s, type, name, elements): # Re-raise as Python ValueError raise ValueError(msg) from None else: # pragma: no cover - raise RuntimeError(str(e)) from None + _raise_jexception(e) self.cache_invalidate(s, type, name) @@ -772,7 +808,7 @@ def set_meta(self, s, name, value): _type = {int: 'Num', float: 'Num', str: 'Str', bool: 'Bool'}[_type] method_name = 'setMeta' + _type except KeyError: - raise TypeError('Cannot store metadata of type {}'.format(_type)) + raise TypeError(f'Cannot store metadata of type {_type}') getattr(self.jindex[s], method_name)(name, value) @@ -818,12 +854,14 @@ def _get_item(self, s, ix_type, name, load=True): try: return getattr(self.jindex[s], f'get{ix_type.title()}')(*args) except java.IxException as e: - if re.match('No item [^ ]* exists in this Scenario', e.args[0]): + # Regex for similar but not consistent messages from Java code + msg = (f"No (item|{ix_type.title()}) '?{name}'? exists in this " + "Scenario!") + if re.match(msg, e.args[0]): # Re-raise as a Python KeyError - raise KeyError(f'No {ix_type.title()} {name!r} exists in this ' - 'Scenario!') from None + raise KeyError(name) from None else: # pragma: no cover - raise RuntimeError('unhandled Java exception') from e + _raise_jexception(e) def start_jvm(jvmargs=None): @@ -864,9 +902,9 @@ def start_jvm(jvmargs=None): convertStrings=True, ) - log.debug('JAVA_HOME: {}'.format(os.environ.get('JAVA_HOME', '(not set)'))) - log.debug('jpype.getDefaultJVMPath: {}'.format(jpype.getDefaultJVMPath())) - log.debug('args to startJVM: {} {}'.format(args, kwargs)) + log.debug(f"JAVA_HOME: {os.environ.get('JAVA_HOME', '(not set)')}") + log.debug(f'jpype.getDefaultJVMPath: {jpype.getDefaultJVMPath()}') + log.debug(f'args to startJVM: {args} {kwargs}') jpype.startJVM(*args, **kwargs) diff --git a/ixmp/cli.py b/ixmp/cli.py index b648e62bb..f17389bef 100644 --- a/ixmp/cli.py +++ b/ixmp/cli.py @@ -1,3 +1,5 @@ +from pathlib import Path + import click import ixmp @@ -5,6 +7,20 @@ ScenarioClass = ixmp.Scenario +class VersionType(click.ParamType): + """A Click parameter type that accepts :class:`int` or 'all'.""" + def convert(self, value, param, ctx): + if value == 'new': + return value + elif isinstance(value, int): + return value + else: + try: + return int(value) + except ValueError: + self.fail(f"{value!r} must be an integer or 'new'") + + @click.group() @click.option('--url', metavar='ixmp://PLATFORM/MODEL/SCENARIO[#VERSION]', help='Scenario URL.') @@ -13,7 +29,7 @@ help='Database properties file.') @click.option('--model', help='Model name.') @click.option('--scenario', help='Scenario name.') -@click.option('--version', type=int, help='Scenario version.') +@click.option('--version', type=VersionType(), help='Scenario version.') @click.pass_context def main(ctx, url, platform, dbprops, model, scenario, version): # Load the indicated Platform @@ -51,6 +67,8 @@ def main(ctx, url, platform, dbprops, model, scenario, version): version=version) except KeyError: pass + except Exception as e: # pragma: no cover + raise click.ClickException(e.args[0]) @main.command() @@ -94,19 +112,79 @@ def config(action, key, value): ixmp.config.save() -@main.command('import') +@main.command() +@click.argument('path', type=click.Path(writable=True)) +@click.pass_obj +def export(context, path): + """Export scenario data to PATH.""" + # NB want to use type=click.Path(..., path_type=Path), but fails on bytes + path = Path(path) + + if not context or 'scen' not in context: + raise click.UsageError('give --url, or --platform, --model, and ' + '--scenario, before export') + + context['scen'].to_excel(path) + + +@main.group('import') +@click.pass_obj +def import_group(context): + """Import time series or scenario data. + + DATA is the path to a file containing input data in CSV (time series only) + or Excel format. + """ + if not context or 'scen' not in context: + raise click.UsageError('give --url, or --platform, --model, and ' + '--scenario, before command import') + + +@import_group.command('timeseries') @click.option('--firstyear', type=int, help='First year of data to include.') @click.option('--lastyear', type=int, help='Final year of data to include.') -@click.argument('data', type=click.Path(exists=True, dir_okay=False)) +@click.argument('file', type=click.Path(exists=True, dir_okay=False)) +@click.pass_obj +def import_timeseries(context, file, firstyear, lastyear): + """Import time series data.""" + context['scen'].read_file(Path(file), firstyear, lastyear) + + +@import_group.command('scenario') +@click.option('--discard-solution', is_flag=True, + help='Discard solution data if necessary.') +@click.option('--add-units', is_flag=True, + help='Add units to the Platform.') +@click.option('--init-items', is_flag=True, + help='Initialize sets and parameters.') +@click.option('--commit-steps', is_flag=True, + help='Commit after each step.') +@click.argument('file', type=click.Path(exists=True, dir_okay=False)) @click.pass_obj -def import_command(context, firstyear, lastyear, data): - """Import time series data. +def import_scenario(context, file, discard_solution, add_units, init_items, + commit_steps): + """Import scenario data.""" + scenario = context['scen'] - DATA is the path to a file containing input data in CSV or Excel format. - """ - from ixmp.utils import import_timeseries + if scenario.has_solution() and discard_solution: + scenario.remove_solution() - import_timeseries(context['scen'], data, firstyear, lastyear) + try: + scenario.check_out() + except ValueError as e: + raise click.ClickException(e.args[0]) # Show exception message to user + except RuntimeError as e: + if 'not yet saved' in e.args[0]: + pass # --version=new; no need to check out + else: # pragma: no cover + raise + + scenario.read_excel( + Path(file), + add_units=add_units, + init_items=init_items, + commit_steps=commit_steps, + ) @main.command() diff --git a/ixmp/core.py b/ixmp/core.py index 0ccac8857..4c5ae52d5 100644 --- a/ixmp/core.py +++ b/ixmp/core.py @@ -1,6 +1,7 @@ from functools import partial from itertools import repeat, zip_longest import logging +from pathlib import Path from warnings import warn import numpy as np @@ -95,21 +96,31 @@ def __getattr__(self, name): raise AttributeError(name) def set_log_level(self, level): - """Set global logger level. + """Set log level for the Platform and its storage :class:`.Backend`. Parameters ---------- level : str - set the logger level if specified, see - https://docs.python.org/3/library/logging.html#logging-levels + Name of a :py:ref:`Python logging level `. """ if level not in dir(logging): msg = '{} not a valid Python logger level, see ' + \ 'https://docs.python.org/3/library/logging.html#logging-level' raise ValueError(msg.format(level)) + log.setLevel(level) logger().setLevel(level) self._backend.set_log_level(level) + def get_log_level(self): + """Return log level of the storage :class:`.Backend`, if any. + + Returns + ------- + str + Name of a :py:ref:`Python logging level `. + """ + return self._backend.get_log_level() + def scenario_list(self, default=True, model=None, scen=None): """Return information about TimeSeries and Scenarios on the Platform. @@ -466,7 +477,7 @@ def preload_timeseries(self): """ self._backend('preload') - def add_timeseries(self, df, meta=False): + def add_timeseries(self, df, meta=False, year_lim=(None, None)): """Add data to the TimeSeries. Parameters @@ -494,11 +505,15 @@ def add_timeseries(self, df, meta=False): If :obj:`True`, store `df` as metadata. Metadata is treated specially when :meth:`Scenario.clone` is called for Scenarios created with ``scheme='MESSAGE'``. + + year_lim : tuple of (int or None, int or None`), optional + Respectively, minimum and maximum years to add from *df*; data for + other years is ignored. """ meta = bool(meta) # Ensure consistent column names - df = to_iamc_template(df) + df = to_iamc_layout(df) if 'value' in df.columns: # Long format; pivot to wide @@ -520,6 +535,15 @@ def add_timeseries(self, df, meta=False): # Columns (year) as integer df.columns = df.columns.astype(int) + # Identify columns to drop + to_drop = set() + if year_lim[0]: + to_drop |= set(filter(lambda y: y < year_lim[0], df.columns)) + if year_lim[1]: + to_drop |= set(filter(lambda y: y > year_lim[1], df.columns)) + + df.drop(to_drop, axis=1, inplace=True) + # Add one time series per row for (r, v, u, sa), data in df.iterrows(): # Values as float; exclude NA @@ -582,7 +606,7 @@ def remove_timeseries(self, df): - `year` """ # Ensure consistent column names - df = to_iamc_template(df) + df = to_iamc_layout(df) id_cols = ['region', 'variable', 'unit', 'subannual'] if 'year' not in df.columns: @@ -648,6 +672,30 @@ def get_geodata(self): .reset_index(drop=True) \ .astype({'meta': 'int64', 'year': 'int64'}) + def read_file(self, path, firstyear=None, lastyear=None): + """Read time series data from a CSV or Microsoft Excel file. + + Parameters + ---------- + path : os.PathLike + File to read. Must have suffix '.csv' or '.xlsx'. + firstyear : int, optional + Only read data from years equal to or later than this year. + lastyear : int, optional + Only read data from years equal to or earlier than this year. + + See also + -------- + .Scenario.read_excel + """ + self.platform._backend.read_file( + Path(path), + ItemType.TS, + filters=dict(scenario=self), + firstyear=firstyear, + lastyear=lastyear, + ) + # %% class Scenario @@ -738,7 +786,7 @@ def from_url(cls, url, errors='warn'): scenario = cls(platform, **scenario_info) except Exception as e: if errors == 'warn': - log.warning('{}: {}\nwhen loading Scenario from url {}' + log.warning('{}: {}\nwhen loading Scenario from url: {!r}' .format(e.__class__.__name__, e.args[0], url)) return None, platform else: @@ -881,6 +929,9 @@ def add_set(self, name, key, comment=None): # TODO expand docstring (here or in doc/source/api.rst) with examples, # per test_core.test_add_set. + if isinstance(key, list) and len(key) == 0: + return # No elements to add + # Get index names for set *name*, may raise KeyError idx_names = self.idx_names(name) @@ -1382,7 +1433,7 @@ def solve(self, model=None, callback=None, cb_kwargs={}, **model_options): 'use `remove_solution()` first!') # Instantiate a model - model = get_model(model, **model_options) + model = get_model(model or self.scheme, **model_options) # Validate *callback* argument if callback is not None and not callable(callback): @@ -1441,41 +1492,94 @@ def set_meta(self, name, value): """ self._backend('set_meta', name, value) + # Input and output + def to_excel(self, path): + """Write Scenario to a Microsoft Excel file. -def to_iamc_template(df): - """Format pd.DataFrame *df* in IAMC style. + Parameters + ---------- + path : os.PathLike + File to write. Must have suffix '.xlsx'. + + See also + -------- + :ref:`excel-data-format` + read_excel + """ + self.platform._backend.write_file(Path(path), ItemType.MODEL, + filters=dict(scenario=self)) + + def read_excel(self, path, add_units=False, init_items=False, + commit_steps=False): + """Read a Microsoft Excel file into the Scenario. + + Parameters + ---------- + path : os.PathLike + File to read. Must have suffix '.xlsx'. + add_units : bool, optional + Add missing units, if any, to the Platform instance. + init_items : bool, optional + Initialize sets and parameters that do not already exist in the + Scenario. + commit_steps : bool, optional + Commit changes after every data addition. + + See also + -------- + :ref:`excel-data-format` + .TimeSeries.read_file + to_excel + """ + self.platform._backend.read_file( + Path(path), + ItemType.MODEL, + filters=dict(scenario=self), + add_units=add_units, + init_items=init_items, + commit_steps=commit_steps, + ) + + +def to_iamc_layout(df): + """Transform *df* to a standard IAMC layout. + + The returned object has: + + - Any (Multi)Index levels reset as columns. + - Lower-case column names 'region', 'variable', 'subannual', and 'unit'. + - If not present in *df*, the value 'Year' in the 'subannual' column. Parameters ---------- - df : :class:`pandas.DataFrame` + df : pandas.DataFrame May have a 'node' column, which will be renamed to 'region'. Returns ------- - :class:`pandas.DataFrame` - The returned object has: - - - Any (Multi)Index levels reset as columns. - - Lower-case column names 'region', 'variable', and 'unit'. + pandas.DataFrame Raises ------ ValueError If 'region', 'variable', or 'unit' is not among the column names. """ - # reset the index if meaningful entries are included there + # Reset the index if meaningful entries are included there if not list(df.index.names) == [None]: df.reset_index(inplace=True) - # rename columns to standard notation + # Rename columns in lower case, and transform 'node' to 'region' cols = {c: str(c).lower() for c in df.columns} cols.update(node='region') df = df.rename(columns=cols) + + required_cols = ['region', 'variable', 'unit'] + missing = list(set(required_cols) - set(df.columns)) + if len(missing): + raise ValueError(f'missing required columns {missing!r}') + + # Add a column 'subannual' with the default value if 'subannual' not in df.columns: df['subannual'] = 'Year' - required_cols = ['region', 'variable', 'unit'] - if not set(required_cols).issubset(set(df.columns)): - missing = list(set(required_cols) - set(df.columns)) - raise ValueError("missing required columns `{}`!".format(missing)) return df diff --git a/ixmp/ixmp.jar b/ixmp/ixmp.jar index 5e166db52..2f6f189a5 100644 Binary files a/ixmp/ixmp.jar and b/ixmp/ixmp.jar differ diff --git a/ixmp/testing.py b/ixmp/testing.py index 4d892c82e..dde7fe3fe 100644 --- a/ixmp/testing.py +++ b/ixmp/testing.py @@ -392,7 +392,8 @@ def test_foo(caplog): for e in expected] if not all(found): missing = [msg for i, msg in enumerate(expected) if not found[i]] - raise AssertionError(f'Did not log {missing}') + raise AssertionError(f'Did not log {missing}\namong:\n' + f'{caplog.messages[first:]}') def assert_qty_equal(a, b, check_attrs=True, **kwargs): diff --git a/ixmp/tests/backend/test_base.py b/ixmp/tests/backend/test_base.py index 0eca463f8..8e50517c9 100644 --- a/ixmp/tests/backend/test_base.py +++ b/ixmp/tests/backend/test_base.py @@ -1,3 +1,5 @@ +from pathlib import Path + import pytest from ixmp.backend import ItemType @@ -62,10 +64,10 @@ class BE2(Backend): # Methods with a default implementation can be called with pytest.raises(NotImplementedError): - be.read_file('path', ItemType.VAR) + be.read_file(Path('foo'), ItemType.VAR) with pytest.raises(NotImplementedError): - be.write_file('path', ItemType.VAR) + be.write_file(Path('foo'), ItemType.VAR) def test_cache_non_hashable(): diff --git a/ixmp/tests/backend/test_jdbc.py b/ixmp/tests/backend/test_jdbc.py index c7c359e8d..f7dd71c05 100644 --- a/ixmp/tests/backend/test_jdbc.py +++ b/ixmp/tests/backend/test_jdbc.py @@ -3,7 +3,7 @@ from pytest import raises import ixmp -from ixmp.testing import assert_logs, make_dantzig +from ixmp.testing import make_dantzig def test_jvm_warn(recwarn): @@ -119,3 +119,27 @@ def test_gh_216(test_mp): # not in set i; but JDBCBackend removes 'beijing' from the filters before # calling the underlying method (https://github.com/iiasa/ixmp/issues/216) scen.par('a', filters=filters) + + +@pytest.fixture +def exception_verbose_true(): + """A fixture which ensures JDBCBackend raises verbose exceptions. + + The set value is not disturbed for other tests/code. + """ + tmp = ixmp.backend.jdbc._EXCEPTION_VERBOSE # Store current value + ixmp.backend.jdbc._EXCEPTION_VERBOSE = True # Ensure True + yield + ixmp.backend.jdbc._EXCEPTION_VERBOSE = tmp # Restore value + + +def test_verbose_exception(test_mp, exception_verbose_true): + # Exception stack trace is logged for debugging + with pytest.raises(RuntimeError) as exc_info: + ixmp.Scenario(test_mp, model='foo', scenario='bar', version=-1) + + exc_msg = exc_info.value.args[0] + assert ("There exists no Scenario 'foo|bar' " + "(version: -1) in the database!" in exc_msg) + assert "at.ac.iiasa.ixmp.database.DbDAO.getRunId" in exc_msg + assert "at.ac.iiasa.ixmp.Platform.getScenario" in exc_msg diff --git a/ixmp/tests/core/test_platform.py b/ixmp/tests/core/test_platform.py index 42d64ec6f..9f8827a5f 100644 --- a/ixmp/tests/core/test_platform.py +++ b/ixmp/tests/core/test_platform.py @@ -1,4 +1,6 @@ """Test all functionality of ixmp.Platform.""" +import logging + import pandas as pd import pytest from pandas.testing import assert_frame_equal @@ -13,16 +15,39 @@ def test_init(): ixmp.Platform(backend='foo') -def test_set_log_level(test_mp): - test_mp.set_log_level('CRITICAL') - test_mp.set_log_level('ERROR') - test_mp.set_log_level('WARNING') - test_mp.set_log_level('INFO') - test_mp.set_log_level('DEBUG') - test_mp.set_log_level('NOTSET') - - with pytest.raises(ValueError): - test_mp.set_log_level(level='foo') +@pytest.fixture +def log_level_mp(test_mp): + """A fixture that preserves the log level of *test_mp*.""" + tmp = test_mp.get_log_level() + yield test_mp + test_mp.set_log_level(tmp) + + +@pytest.mark.parametrize('level, exc', [ + ('CRITICAL', None), + ('ERROR', None), + ('WARNING', None), + ('INFO', None), + ('DEBUG', None), + ('NOTSET', None), + # An unknown string fails + ('FOO', ValueError), + # TODO also support Python standard library values + (logging.CRITICAL, ValueError), + (logging.ERROR, ValueError), + (logging.WARNING, ValueError), + (logging.INFO, ValueError), + (logging.DEBUG, ValueError), + (logging.NOTSET, ValueError), +]) +def test_log_level(log_level_mp, level, exc): + """Log level can be set and retrieved.""" + if exc is None: + log_level_mp.set_log_level(level) + assert log_level_mp.get_log_level() == level + else: + with pytest.raises(exc): + log_level_mp.set_log_level(level) def test_scenario_list(mp): diff --git a/ixmp/tests/core/test_scenario.py b/ixmp/tests/core/test_scenario.py index 73dd0bc00..2882482e8 100644 --- a/ixmp/tests/core/test_scenario.py +++ b/ixmp/tests/core/test_scenario.py @@ -52,22 +52,22 @@ def test_default_version(self, mp): scen = ixmp.Scenario(mp, **models['dantzig']) assert scen.version == 2 - def test_scenario_from_url(self, mp, caplog): - url = 'ixmp://{}/Douglas Adams/Hitchhiker'.format(mp.name) + def test_from_url(self, mp, caplog): + url = f'ixmp://{mp.name}/Douglas Adams/Hitchhiker' # Default version is loaded scen, mp = ixmp.Scenario.from_url(url) assert scen.version == 1 # Giving an invalid version with errors='raise' raises an exception - with pytest.raises(Exception, match='There was a problem getting the ' - 'run id from the database!'): + expected = ("There exists no Scenario 'Douglas Adams|Hitchhiker' " + "(version: 10000) in the database!") + with pytest.raises(Exception, match=expected): scen, mp = ixmp.Scenario.from_url(url + '#10000', errors='raise') # Giving an invalid scenario with errors='warn' raises an exception - msg = ("RuntimeError: There was a problem getting 'Hitchhikerfoo' in " - "table 'SCENARIO' from the database!\nwhen loading Scenario " - f"from url {url}") + msg = ("ValueError: scenario='Hitchhikerfoo'\nwhen loading Scenario " + f"from url: {(url + 'foo')!r}") with assert_logs(caplog, msg): scen, mp = ixmp.Scenario.from_url(url + 'foo') assert scen is None and isinstance(mp, ixmp.Platform) @@ -192,6 +192,83 @@ def test_load_scenario_data_clear_cache(self, mp): scen.load_scenario_data() scen.platform._backend.cache_invalidate(scen, 'par', 'd') + # I/O + def test_excel_io(self, scen, scen_empty, tmp_path, caplog): + tmp_path /= 'output.xlsx' + + # FIXME remove_solution, check_out, commit, solve, commit should not + # be needed to make this small data addition. + scen.remove_solution() + scen.check_out() + + # A 1-D set indexed by another set + scen.init_set('foo', 'j') + scen.add_set('foo', [['new-york'], ['topeka']]) + # A scalar parameter with unusual units + scen.platform.add_unit('pounds') + scen.init_scalar('bar', 100, 'pounds') + # A parameter with no values + scen.init_par('baz_1', ['i', 'j']) + # A parameter with ambiguous index name + scen.init_par('baz_2', ['i'], ['i_dim']) + scen.add_par('baz_2', dict(value=[1.1], i_dim=['seattle'])) + # A 2-D set with ambiguous index names + scen.init_set('baz_3', ['i', 'i'], ['i', 'i_also']) + scen.add_set('baz_3', [['seattle', 'seattle']]) + + scen.commit('') + scen.solve() + + # Solved Scenario can be written to file + scen.to_excel(tmp_path) + + # With init_items=False, can't be read into an empty Scenario + with pytest.raises(ValueError, match="no set 'i'; " + "try init_items=True"): + scen_empty.read_excel(tmp_path) + + # File can be read with init_items=True + scen_empty.read_excel(tmp_path, init_items=True, commit_steps=True) + + # Contents of the Scenarios are the same, except for unreadable items + assert set(scen_empty.par_list()) | {'baz_1', 'baz_2'} \ + == set(scen.par_list()) + assert set(scen_empty.set_list()) | {'baz_3'} == set(scen.set_list()) + assert_frame_equal(scen_empty.set('foo'), scen.set('foo')) + # NB could make a more exact comparison of the Scenarios + + # Pre-initialize skipped items 'baz_2' and 'baz_3' + scen_empty.init_par('baz_2', ['i'], ['i_dim']) + scen_empty.init_set('baz_3', ['i', 'i'], ['i', 'i_also']) + + # Data can be read into an existing Scenario without init_items or + # commit_steps arguments + scen_empty.read_excel(tmp_path) + + # Re-initialize an item with different index names + scen_empty.remove_par('d') + scen_empty.init_par('d', idx_sets=['i', 'j'], idx_names=['I', 'J']) + + # Reading now logs an error about conflicting dims + with assert_logs(caplog, "Existing par 'd' has index names(s)"): + scen_empty.read_excel(tmp_path, init_items=True) + + # A new, empty Platform (different from the one under scen -> mp -> + # test_mp) that lacks all units + mp = ixmp.Platform(backend='jdbc', driver='hsqldb', + url='jdbc:hsqldb:mem:excel_io') + # A Scenario without the 'dantzig' scheme -> no contents at all + s = ixmp.Scenario(mp, model='foo', scenario='bar', scheme='empty', + version='new') + + # Fails with add_units=False + with pytest.raises(ValueError, match="The unit 'pounds' does not exist" + " in the database!"): + s.read_excel(tmp_path, init_items=True) + + # Succeeds with add_units=True + s.read_excel(tmp_path, add_units=True, init_items=True) + # Combined tests def test_meta(self, mp): test_dict = { @@ -259,8 +336,7 @@ def test_set(scen_empty): scen = scen_empty # Add element to a non-existent set - with pytest.raises(KeyError, - match="No Item 'foo' exists in this Scenario!"): + with pytest.raises(KeyError, match=repr('foo')): scen.add_set('foo', 'bar') # Initialize a 0-D set diff --git a/ixmp/tests/core/test_timeseries.py b/ixmp/tests/core/test_timeseries.py index f3085551a..2956b6c49 100644 --- a/ixmp/tests/core/test_timeseries.py +++ b/ixmp/tests/core/test_timeseries.py @@ -280,7 +280,6 @@ def test_edit_with_region_synonyms(mp, ts, cls): info = dict(model=ts.model, scenario=ts.scenario) exp = expected(DATA[0], ts) - mp.set_log_level('DEBUG') mp.add_region_synonym('Hell', 'World') ts.add_timeseries(DATA[0]) @@ -509,7 +508,6 @@ def test_timeseries_edit_iamc(mp): def test_timeseries_edit_with_region_synonyms(mp): args_all = ('Douglas Adams 1', 'test_remove_all') - mp.set_log_level('DEBUG') mp.add_region_synonym('Hell', 'World') scen = prepare_scenario(mp, args_all) obs = scen.timeseries() diff --git a/ixmp/tests/test_access.py b/ixmp/tests/test_access.py index 1e445a43e..352d34f70 100644 --- a/ixmp/tests/test_access.py +++ b/ixmp/tests/test_access.py @@ -61,7 +61,6 @@ def test_check_single_model_access(mock, tmp_path, test_data_path): auth_url=mock.pretend_url) mp = ixmp.Platform(backend='jdbc', dbprops=test_props) - mp.set_log_level('DEBUG') granted = mp.check_access('test_user', 'test_model') assert granted @@ -91,7 +90,6 @@ def test_check_multi_model_access(mock, tmp_path, test_data_path): auth_url=mock.pretend_url) mp = ixmp.Platform(backend='jdbc', dbprops=test_props) - mp.set_log_level('DEBUG') access = mp.check_access('test_user', ['test_model', 'non_existing_model']) assert access['test_model'] diff --git a/ixmp/tests/test_cli.py b/ixmp/tests/test_cli.py index 53a430e6b..eba8c119e 100644 --- a/ixmp/tests/test_cli.py +++ b/ixmp/tests/test_cli.py @@ -1,10 +1,28 @@ from pathlib import Path from pandas.testing import assert_frame_equal -from click.exceptions import UsageError +from click.exceptions import BadParameter, UsageError import ixmp +from ixmp.cli import VersionType from ixmp.testing import models, populate_test_platform import pandas as pd +import pytest + + +def test_versiontype(): + vt = VersionType() + # str converts to int + assert vt.convert('1', None, None) == 1 + assert vt.convert('-1', None, None) == -1 + + # str 'new' is passes through + assert vt.convert('new', None, None) == 'new' + + # int passes through + assert vt.convert(1, None, None) == 1 + + with pytest.raises(BadParameter, match="'xx' must be an integer or 'new'"): + vt.convert('xx', None, None) def test_main(ixmp_cli, test_mp, tmp_path): @@ -128,7 +146,7 @@ def call(*args, exit_0=True): assert r.exit_code == 1 -def test_import(ixmp_cli, test_mp, test_data_path): +def test_import_ts(ixmp_cli, test_mp, test_data_path): # Ensure the 'canning problem'/'standard' TimeSeries exists populate_test_platform(test_mp) @@ -138,11 +156,12 @@ def test_import(ixmp_cli, test_mp, test_data_path): '--model', models['dantzig']['model'], '--scenario', models['dantzig']['scenario'], '--version', '1', - 'import', + 'import', 'timeseries', '--firstyear', '2020', + '--lastyear', '2200', str(test_data_path / 'timeseries_canning.csv'), ]) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output # Expected data exp = pd.DataFrame.from_dict({ @@ -165,6 +184,61 @@ def test_import(ixmp_cli, test_mp, test_data_path): assert len(scen.timeseries(variable=['Testing'])) == 0 +def test_excel_io(ixmp_cli, test_mp, tmp_path): + populate_test_platform(test_mp) + tmp_path /= 'dantzig.xlsx' + + # Invoke the CLI to export data to Excel + cmd = [ + '--platform', test_mp.name, + '--model', models['dantzig']['model'], + '--scenario', models['dantzig']['scenario'], + 'export', str(tmp_path), + ] + result = ixmp_cli.invoke(cmd) + assert result.exit_code == 0, result.output + + # Fails without platform/scenario info + assert ixmp_cli.invoke(cmd[6:]).exit_code == UsageError.exit_code + + # Invoke the CLI to read data from Excel + cmd = [ + '--platform', test_mp.name, + '--model', models['dantzig']['model'], + '--scenario', models['dantzig']['scenario'], + 'import', 'scenario', str(tmp_path), + ] + + # Fails without platform/scenario info + assert ixmp_cli.invoke(cmd[6:]).exit_code == UsageError.exit_code + + # Fails without --discard-solution + result = ixmp_cli.invoke(cmd) + assert result.exit_code == 1 + assert 'This Scenario has a solution' in result.output + + # Succeeds with --discard-solution + cmd.insert(-1, '--discard-solution') + result = ixmp_cli.invoke(cmd) + assert result.exit_code == 0, result.output + + # Import into a new model name fails without --init-items + cmd = [ + '--platform', test_mp.name, + '--model', 'foo model', + '--scenario', 'bar scenario', + '--version', 'new', + 'import', 'scenario', str(tmp_path), + ] + result = ixmp_cli.invoke(cmd) + assert result.exit_code == 1 + + # Succeeds + cmd.insert(-1, '--init-items') + result = ixmp_cli.invoke(cmd) + assert result.exit_code == 0, result.output + + def test_report(ixmp_cli): # 'report' without specifying a platform/scenario is a UsageError result = ixmp_cli.invoke(['report', 'key']) diff --git a/ixmp/tests/test_utils.py b/ixmp/tests/test_utils.py index 9f1ce5c2e..463a510ad 100644 --- a/ixmp/tests/test_utils.py +++ b/ixmp/tests/test_utils.py @@ -1,6 +1,4 @@ """Tests for ixmp.utils.""" -import pandas as pd -from pandas.testing import assert_frame_equal import pytest from pytest import mark, param @@ -8,53 +6,6 @@ from ixmp.testing import populate_test_platform -def make_obs(fname, exp, **kwargs): - utils.pd_write(exp, fname, index=False) - obs = utils.pd_read(fname, **kwargs) - return obs - - -def test_pd_io_csv(tmp_path): - - fname = tmp_path / "test.csv" - exp = pd.DataFrame({'a': [0, 1], 'b': [2, 3]}) - obs = make_obs(fname, exp) - assert_frame_equal(obs, exp) - - -def test_pd_io_xlsx(tmp_path): - - fname = tmp_path / "test.xlsx" - exp = pd.DataFrame({'a': [0, 1], 'b': [2, 3]}) - obs = make_obs(fname, exp) - assert_frame_equal(obs, exp) - - -def test_pd_io_xlsx_multi(tmp_path): - - fname = tmp_path / "test.xlsx" - exp = { - 'sheet1': pd.DataFrame({'a': [0, 1], 'b': [2, 3]}), - 'sheet2': pd.DataFrame({'c': [4, 5], 'd': [6, 7]}), - } - obs = make_obs(fname, exp, sheet_name=None) - for k, _exp in exp.items(): - _obs = obs[k] - assert_frame_equal(_obs, _exp) - - -def test_pd_write(tmp_path): - - fname = 'test.csv' - d = tmp_path / "sub" - d.mkdir() - - data_frame = [1, 2, 3, 4] - - with pytest.raises(ValueError): - assert utils.pd_write(data_frame, fname) - - def test_check_year(): # If y is a string value, raise a Value Error. diff --git a/ixmp/utils.py b/ixmp/utils.py index 7ec9a3f56..4af888f21 100644 --- a/ixmp/utils.py +++ b/ixmp/utils.py @@ -4,7 +4,6 @@ from urllib.parse import urlparse import pandas as pd -from pathlib import Path # globally accessible logger @@ -119,33 +118,6 @@ def parse_url(url): return platform_info, scenario_info -def pd_read(f, *args, **kwargs): - """Try to read a file with pandas, no fancy stuff""" - f = Path(f) - if f.suffix == '.csv': - return pd.read_csv(f, *args, **kwargs) - else: - return pd.read_excel(f, *args, **kwargs) - - -def pd_write(df, f, *args, **kwargs): - """Try to write one or more dfs with pandas, no fancy stuff""" - f = Path(f) - is_pd = isinstance(df, (pd.DataFrame, pd.Series)) - if f.suffix == '.csv': - if not is_pd: - raise ValueError('Must pass a Dataframe if using csv files') - df.to_csv(f, *args, **kwargs) - else: - writer = pd.ExcelWriter(f, engine='xlsxwriter') - if is_pd: - sheet_name = kwargs.pop('sheet_name', 'Sheet1') - df = {sheet_name: df} - for k, v in df.items(): - v.to_excel(writer, sheet_name=k, *args, **kwargs) - writer.save() - - def year_list(x): """Return the elements of x that can be cast to year (int).""" lst = [] @@ -170,31 +142,6 @@ def filtered(df, filters): return df[mask] -def import_timeseries(scenario, data, firstyear=None, lastyear=None): - """Import from a *data* file into *scenario*.""" - df = pd_read(data) - df = df.rename(columns={c: str(c).lower() for c in df.columns}) - - cols = year_list(df.columns) - years = [int(i) for i in cols] - fyear = int(firstyear or min(years)) - lyear = int(lastyear or max(years)) - cols = [c for c in cols if int(c) >= fyear and int(c) <= lyear] - df = df[['region', 'variable', 'unit'] + cols] - df.region = [x if x == 'World' else 'R11_' + x for x in df.region] - - scenario.check_out(timeseries_only=True) - scenario.add_timeseries(df) - - annot = 'adding timeseries data from file {}'.format(data) - if firstyear is not None: - annot = '{} from {}'.format(annot, firstyear) - if lastyear is not None: - annot = '{} until {}'.format(annot, lastyear) - - scenario.commit(annot) - - def format_scenario_list(platform, model=None, scenario=None, match=None, default_only=False, as_url=False): """Return a formatted list of TimeSeries on *platform*. diff --git a/setup.py b/setup.py index 6a6c3d428..c98e1b76e 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ EXTRAS_REQUIRE = { 'tests': ['codecov', 'jupyter', 'pretenders>=1.4.4', 'pytest-cov', - 'pytest>=3.9', 'sparse'], + 'pytest>=5', 'sparse'], 'docs': ['numpydoc', 'sphinx', 'sphinx_rtd_theme', 'sphinxcontrib-bibtex'], 'tutorial': ['jupyter'], }