From 25b51daf25d9fe989fb27351a9134c48eb427623 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 8 Jun 2022 15:40:45 +0200 Subject: [PATCH 1/8] Added support for custom location of custom CMOR tables --- esmvalcore/cmor/table.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 4c5bd2e1ff..60a863dd1a 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -55,11 +55,22 @@ def read_cmor_tables(cfg_developer=None): with open(var_alt_names_file, 'r') as yfile: alt_names = yaml.safe_load(yfile) - custom = CustomInfo() CMOR_TABLES.clear() + + # Try to infer location for custom tables from config-developer.yml file, + # if not possible, use default location + custom_path = None + if 'custom' in cfg_developer: + custom_path = cfg_developer['custom'].get('cmor_path') + if custom_path is not None: + custom_path = os.path.expandvars(os.path.expanduser(custom_path)) + custom = CustomInfo(custom_path) CMOR_TABLES['custom'] = custom + install_dir = os.path.dirname(os.path.realpath(__file__)) for table in cfg_developer: + if table == 'custom': + continue CMOR_TABLES[table] = _read_table(cfg_developer, table, install_dir, custom, alt_names) @@ -815,18 +826,40 @@ class CustomInfo(CMIP5Info): Full path to the table or name for the table if it is present in ESMValTool repository """ + def __init__(self, cmor_tables_path=None): cwd = os.path.dirname(os.path.realpath(__file__)) - self._cmor_folder = os.path.join(cwd, 'tables', 'custom') + default_cmor_folder = os.path.join(cwd, 'tables', 'custom') + + # Get custom location of CMOR tables if possible + if cmor_tables_path is None: + self._cmor_folder = default_cmor_folder + else: + self._cmor_folder = self._get_cmor_path(cmor_tables_path) + if not os.path.isdir(self._cmor_folder): + raise ValueError(f"Custom CMOR tables path {self._cmor_folder} is " + f"not a directory") + self.tables = {} self.var_to_freq = {} table = TableInfo() table.name = 'custom' self.tables[table.name] = table - self._coordinates_file = os.path.join( + + # Try to read coordinates from custom location, use default location if + # not possible + coordinates_file = os.path.join( self._cmor_folder, 'CMOR_coordinates.dat', ) + if os.path.isfile(coordinates_file): + self._coordinates_file = coordinates_file + else: + self._coordinates_file = os.path.join( + default_cmor_folder, + 'CMOR_coordinates.dat', + ) + self.coords = {} self._read_table_file(self._coordinates_file, self.tables['custom']) for dat_file in glob.glob(os.path.join(self._cmor_folder, '*.dat')): From 94b39272818661531dbfbd4ecd26c7d5ae2ade4e Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 15 Jun 2022 10:16:59 +0200 Subject: [PATCH 2/8] Added doc for using custom location for custom CMOR tables --- doc/quickstart/configure.rst | 68 +++++++++++++++++++++++++++++++-- doc/quickstart/find_data.rst | 6 +++ esmvalcore/config-developer.yml | 16 ++++++-- 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 14a6d4a21f..0116f17277 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -510,9 +510,10 @@ related to CMOR table settings available: * ``cmor_type``: can be ``CMIP5`` if the CMOR table is in the same format as the CMIP5 table or ``CMIP6`` if the table is in the same format as the CMIP6 table. * ``cmor_strict``: if this is set to ``false``, the CMOR table will be - extended with variables from the ``esmvalcore/cmor/tables/custom`` directory - and it is possible to use variables with a ``mip`` which is different from - the MIP table in which they are defined. + extended with variables from the :ref:`custom_cmor_tables` (by default loaded + from the ``esmvalcore/cmor/tables/custom`` directory) and it is possible to + use variables with a ``mip`` which is different from the MIP table in which + they are defined. * ``cmor_path``: path to the CMOR table. Relative paths are with respect to `esmvalcore/cmor/tables`_. Defaults to the value provided in ``cmor_type`` written in lower case. @@ -520,6 +521,61 @@ related to CMOR table settings available: to get the name of the file containing the ``mip`` table. Defaults to the value provided in ``cmor_type``. +.. _custom_cmor_tables: + +Custom CMOR tables +------------------ + +As mentioned in the previous section, the CMOR tables of projects that use +``cmor_strict: false`` will be extended with custom CMOR tables. +By default, these are loaded from `esmvalcore/cmor/tables/custom`_. +However, by using the special project ``custom`` in the +``config-developer.yml`` file with the option ``cmor_path```, a custom location +for these custom CMOR tables can be specified: + +.. code-block:: yaml + + custom: + cmor_path: ~/my/own/custom_tables + +This path can be given as relative path (relative to `esmvalcore/cmor/tables`_) +or as absolute path. +Other options given for this special table will be ignored. + +Custom tables in this directory need to follow the naming convention +``CMOR_{short_name}.dat`` and need to be given in CMIP5 format. + +Example for the file ``CMOR_asr.dat``: + +.. code-block:: + + SOURCE: CMIP5 + !============ + variable_entry: asr + !============ + modeling_realm: atmos + !---------------------------------- + ! Variable attributes: + !---------------------------------- + standard_name: + units: W m-2 + cell_methods: time: mean + cell_measures: area: areacella + long_name: Absorbed shortwave radiation + !---------------------------------- + ! Additional variable information: + !---------------------------------- + dimensions: longitude latitude time + type: real + positive: down + !---------------------------------- + ! + +It is also possible to use a special coordinates file ``CMOR_coordinates.dat``. +If this is not present in the custom directory, the one from the default +directory (`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat`_) is used. + + .. _filterwarnings_config-developer: Filter preprocessor warnings @@ -590,6 +646,12 @@ Example: A detailed description on how to add support for further native datasets is given :ref:`here `. +.. hint:: + + When using data from native model output, it might be helpful to specify a + custom location for the :ref:`custom_cmor_tables`. + This allows allows reading arbitrary variables from native model output. + .. _config-ref: diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 42d7bd3b33..80a4cf4685 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -88,6 +88,12 @@ project, e.g., ``ICON`` (mostly native models). A detailed description of how to include new native datasets is given :ref:`here `. +.. hint:: + + When using data from native model output, it might be helpful to specify a + custom location for the :ref:`custom_cmor_tables`. + This allows allows reading arbitrary variables from native model output. + .. _read_native_obs: Supported native reanalysis/observational datasets diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index d3fd46013a..0d033d7f52 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -2,8 +2,8 @@ # Developer's configuration file for the ESMValTool ############################################################################### # This file retains the project- and machine-dependent directory and file name -# definitions of the input and output data -# Each dictionary is structured as follows +# definitions of the input and output data. +# Each dictionary is structured as follows: # # PROJECT: # input_dir: @@ -14,10 +14,18 @@ # input_file: # output_file: # -# Only the default drs is mandatory, the others are optional +# Only the default drs is mandatory, the others are optional. +# +# In addition, an entry for the custom tables can be given. For this, only the +# option 'cmor_path' is considered, which specifies the directory from which +# custom CMOR tables are loaded. 'cmor_path' can be a relative path (relative +# to ESMValCore/esmvalcore/cmor/tables) or an absolute path. By default, uses +# ESMValCore/esmvalcore/cmor/tables/custom. +# +# custom: +# cmor_path: ~/my/own/custom_tables ############################################################################### --- - CMIP6: cmor_strict: true input_dir: From 1bf522b7b8669695995341253796e538a148ff1e Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 15 Jun 2022 10:18:27 +0200 Subject: [PATCH 3/8] Added missing space in config-developer.yml file --- esmvalcore/config-developer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 0d033d7f52..8c7dedcec0 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -26,6 +26,7 @@ # cmor_path: ~/my/own/custom_tables ############################################################################### --- + CMIP6: cmor_strict: true input_dir: From ce32ca7e1aad020a23ac4025b5ec379c81f87623 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 15 Jun 2022 10:27:22 +0200 Subject: [PATCH 4/8] Fixed doc --- doc/quickstart/configure.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 0116f17277..3917a3a87f 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -528,7 +528,8 @@ Custom CMOR tables As mentioned in the previous section, the CMOR tables of projects that use ``cmor_strict: false`` will be extended with custom CMOR tables. -By default, these are loaded from `esmvalcore/cmor/tables/custom`_. +By default, these are loaded from `esmvalcore/cmor/tables/custom +`_. However, by using the special project ``custom`` in the ``config-developer.yml`` file with the option ``cmor_path```, a custom location for these custom CMOR tables can be specified: @@ -573,7 +574,9 @@ Example for the file ``CMOR_asr.dat``: It is also possible to use a special coordinates file ``CMOR_coordinates.dat``. If this is not present in the custom directory, the one from the default -directory (`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat`_) is used. +directory (`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat +`_) +is used. .. _filterwarnings_config-developer: From a4fb5aed7bf5ebc4a90f5e502bd2f16a4dbaa4d9 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 15 Jun 2022 11:05:22 +0200 Subject: [PATCH 5/8] Optimized doc --- doc/develop/fixing_data.rst | 10 ++++++++++ doc/quickstart/configure.rst | 7 +++++-- doc/quickstart/find_data.rst | 5 ++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 6176ee5e3e..2cd0d4eaf8 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -342,6 +342,16 @@ This section describes how to add support for additional native datasets. You can choose to host this new data source either under a dedicated project or under project ``native6``. +.. _add_new_fix_native_datasets_config: + +Configuration +------------- +An example of a configuration in ``config-developer.yml`` for projects used for +native datasets is given :ref:`here `. +Make sure to use the option ``cmor_strict: false`` for these projects if you +want to make use of :ref:`custom_cmor_tables`. +This allows reading arbitrary variables from native model output. + .. _add_new_fix_native_datasets_locate_data: Locate data diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 3917a3a87f..8746e52bc8 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -531,7 +531,7 @@ As mentioned in the previous section, the CMOR tables of projects that use By default, these are loaded from `esmvalcore/cmor/tables/custom `_. However, by using the special project ``custom`` in the -``config-developer.yml`` file with the option ``cmor_path```, a custom location +``config-developer.yml`` file with the option ``cmor_path``, a custom location for these custom CMOR tables can be specified: .. code-block:: yaml @@ -653,7 +653,10 @@ given :ref:`here `. When using data from native model output, it might be helpful to specify a custom location for the :ref:`custom_cmor_tables`. - This allows allows reading arbitrary variables from native model output. + This allows reading arbitrary variables from native model output. + Note that this requires the option ``cmor_strict: false`` in the + :ref:`project configuration ` used for the native + model output. .. _config-ref: diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 80a4cf4685..5dc366f18a 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -92,7 +92,10 @@ A detailed description of how to include new native datasets is given When using data from native model output, it might be helpful to specify a custom location for the :ref:`custom_cmor_tables`. - This allows allows reading arbitrary variables from native model output. + This allows reading arbitrary variables from native model output. + Note that this requires the option ``cmor_strict: false`` in the + :ref:`project configuration ` used for the native + model output. .. _read_native_obs: From 6c5b6e6c7695bcdfa698fa8be9179445744e9f82 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 15 Jun 2022 12:16:35 +0200 Subject: [PATCH 6/8] Added tests --- doc/develop/fixing_data.rst | 1 + .../integration/cmor/test_read_cmor_tables.py | 29 +++++++++++++++ tests/integration/cmor/test_table.py | 35 ++++++++++++++++++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 2cd0d4eaf8..23e039ab35 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -346,6 +346,7 @@ under project ``native6``. Configuration ------------- + An example of a configuration in ``config-developer.yml`` for projects used for native datasets is given :ref:`here `. Make sure to use the option ``cmor_strict: false`` for these projects if you diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index b74feaa7ed..8c520f8847 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -5,6 +5,17 @@ from esmvalcore.cmor.table import __file__ as root from esmvalcore.cmor.table import read_cmor_tables +CUSTOM_CFG_DEVELOPER = { + 'custom': {'cmor_path': Path(root).parent / 'tables' / 'custom'}, + 'CMIP6': { + 'cmor_strict': True, + 'input_dir': {'default': '/'}, + 'input_file': '*.nc', + 'output_file': 'out.nc', + 'cmor_type': 'CMIP6', + }, +} + def test_read_cmor_tables(): """Test that the function `read_cmor_tables` loads the tables correctly.""" @@ -33,3 +44,21 @@ def test_read_cmor_tables(): table = CMOR_TABLES[project] assert Path(table._cmor_folder) == table_path / 'obs4mips' / 'Tables' assert table.strict is False + + +def test_read_custom_cmor_tables(): + """Test reading of custom CMOR tables.""" + read_cmor_tables(CUSTOM_CFG_DEVELOPER) + + assert len(CMOR_TABLES) == 2 + assert 'CMIP6' in CMOR_TABLES + assert 'custom' in CMOR_TABLES + + custom_table = CMOR_TABLES['custom'] + assert (Path(custom_table._cmor_folder) == + Path(root).parent / 'tables' / 'custom') + assert (Path(custom_table._coordinates_file) == + Path(root).parent / 'tables' / 'custom' / 'CMOR_coordinates.dat') + + cmip6_table = CMOR_TABLES['CMIP6'] + assert cmip6_table.default is custom_table diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 568dd6aaba..db8ce81510 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -347,12 +347,45 @@ def setUpClass(cls): """ cls.variables_info = CustomInfo() + def test_custom_tables_default_location(self): + """Test constructor with default tables location.""" + custom_info = CustomInfo() + expected_cmor_folder = os.path.join( + os.path.dirname(esmvalcore.cmor.__file__), + 'tables', + 'custom', + ) + expected_coordinate_file = os.path.join( + os.path.dirname(esmvalcore.cmor.__file__), + 'tables', + 'custom', + 'CMOR_coordinates.dat', + ) + self.assertEqual(custom_info._cmor_folder, expected_cmor_folder) + self.assertEqual(custom_info._coordinates_file, + expected_coordinate_file) + def test_custom_tables_location(self): """Test constructor with custom tables location.""" cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__)) cmor_tables_path = os.path.join(cmor_path, 'tables', 'cmip5') cmor_tables_path = os.path.abspath(cmor_tables_path) - CustomInfo(cmor_tables_path) + custom_info = CustomInfo(cmor_tables_path) + self.assertEqual(custom_info._cmor_folder, cmor_tables_path) + + expected_coordinate_file = os.path.join( + os.path.dirname(esmvalcore.cmor.__file__), + 'tables', + 'custom', + 'CMOR_coordinates.dat', + ) + self.assertEqual(custom_info._coordinates_file, + expected_coordinate_file) + + def test_custom_tables_invalid_location(self): + """Test constructor with invalid custom tables location.""" + with self.assertRaises(ValueError): + CustomInfo('this_file_does_not_exist.dat') def test_get_variable_netcre(self): """Get tas variable.""" From f82d80bce3346e9491316729a18c8fbbeb1df79a Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 15 Jun 2022 13:40:33 +0200 Subject: [PATCH 7/8] Fixed tests --- tests/integration/cmor/test_read_cmor_tables.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index 8c520f8847..8c4169cbc8 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -62,3 +62,6 @@ def test_read_custom_cmor_tables(): cmip6_table = CMOR_TABLES['CMIP6'] assert cmip6_table.default is custom_table + + # Restore default tables + read_cmor_tables(read_config_developer_file()) From c9943b3bbe42f19315e6cba00a06471cb25a966c Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Wed, 15 Jun 2022 15:05:45 +0200 Subject: [PATCH 8/8] Renamed native models -> native datasets --- doc/develop/fixing_data.rst | 2 +- doc/quickstart/configure.rst | 8 ++++---- doc/quickstart/find_data.rst | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index 23e039ab35..ddfc5978fb 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -351,7 +351,7 @@ An example of a configuration in ``config-developer.yml`` for projects used for native datasets is given :ref:`here `. Make sure to use the option ``cmor_strict: false`` for these projects if you want to make use of :ref:`custom_cmor_tables`. -This allows reading arbitrary variables from native model output. +This allows reading arbitrary variables from native datasets. .. _add_new_fix_native_datasets_locate_data: diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 8746e52bc8..0016982308 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -586,7 +586,7 @@ Filter preprocessor warnings It is possible to ignore specific warnings of the preprocessor for a given ``project``. -This is particularly useful for native models which do not follow the CMOR +This is particularly useful for native datasets which do not follow the CMOR standard by default and consequently produce a lot of warnings when handled by Iris. This can be configured in the ``config-developer.yml`` file for some steps of @@ -651,9 +651,9 @@ given :ref:`here `. .. hint:: - When using data from native model output, it might be helpful to specify a - custom location for the :ref:`custom_cmor_tables`. - This allows reading arbitrary variables from native model output. + When using native datasets, it might be helpful to specify a custom location + for the :ref:`custom_cmor_tables`. + This allows reading arbitrary variables from native datasets. Note that this requires the option ``cmor_strict: false`` in the :ref:`project configuration ` used for the native model output. diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 5dc366f18a..134d49d6ae 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -90,9 +90,9 @@ A detailed description of how to include new native datasets is given .. hint:: - When using data from native model output, it might be helpful to specify a - custom location for the :ref:`custom_cmor_tables`. - This allows reading arbitrary variables from native model output. + When using native datasets, it might be helpful to specify a custom location + for the :ref:`custom_cmor_tables`. + This allows reading arbitrary variables from native datasets. Note that this requires the option ``cmor_strict: false`` in the :ref:`project configuration ` used for the native model output.