Support CMIP3

.gitignore

@@ -1,3 +1,6 @@
+# Autogenerated files
+_sidebar.rst.inc
+
 # Distribution / packaging
 .Python
 build/

esmvalcore/_data_finder.py

@@ -10,6 +10,8 @@
 import re
 import glob
 
+import iris
+
 from ._config import get_project_config
 
 logger = logging.getLogger(__name__)
@@ -46,8 +48,8 @@ def get_start_end_year(filename):
     """
     name = os.path.splitext(filename)[0]
 
-    filename = name.split(os.sep)[-1]
-    filename_list = [elem.split('-') for elem in filename.split('_')]
+    name = name.split(os.sep)[-1]
+    filename_list = [elem.split('-') for elem in name.split('_')]
     filename_list = [elem for sublist in filename_list for elem in sublist]
 
     pos_ydates = [elem.isdigit() and len(elem) >= 4 for elem in filename_list]
@@ -67,16 +69,35 @@ def get_start_end_year(filename):
         filename_list[ind] for ind, _ in enumerate(pos_ydates)
         if pos_ydates_r[ind] or pos_ydates_l[ind]
     ]
-
+    start_year = None
+    end_year = None
     if len(dates) == 1:
         start_year = int(dates[0][:4])
         end_year = start_year
     elif len(dates) == 2:
         start_year, end_year = int(dates[0][:4]), int(dates[1][:4])
     else:
-        raise ValueError('Name {0} dates do not match a recognized '
-                         'pattern'.format(name))
-
+        # Slower than just parsing the name
+        try:
+            cubes = iris.load(filename)
+        except OSError:
+            raise ValueError('File {0} can not be read'.format(filename))
+
+        for cube in cubes:
+            logger.debug(cube)
+            try:
+                time = cube.coord('time')
+            except iris.exceptions.CoordinateNotFoundError:
+                continue
+            start_year = time.cell(0).point.year
+            end_year = time.cell(-1).point.year
+            break
+
+    if not start_year or not end_year:
+        raise ValueError(
+            'File {0} dates do not match a recognized pattern and time can '
+            'not be readed from the file'.format(filename)
+        )
     return start_year, end_year
 
 

esmvalcore/_recipe.py

@@ -1002,12 +1002,14 @@ def _initialize_variables(self, raw_variable, raw_datasets):
             required_keys.update({'start_year', 'end_year'})
         for variable in variables:
             _update_from_others(variable, ['cmor_table', 'mip'], datasets)
-            institute = get_institutes(variable)
-            if institute:
-                variable['institute'] = institute
-            activity = get_activity(variable)
-            if activity:
-                variable['activity'] = activity
+            if 'institute' not in variable:
+                institute = get_institutes(variable)
+                if institute:
+                    variable['institute'] = institute
+            if 'activity' not in variable:
+                activity = get_activity(variable)
+                if activity:
+                    variable['activity'] = activity
             check.variable(variable, required_keys)
         variables = self._expand_ensemble(variables)
         return variables

esmvalcore/cmor/check.py

@@ -315,7 +315,6 @@ def _check_dim_names(self):
                             coord.var_name = coordinate.out_name
                         else:
                             self.report_error(
-                                'Coordinate {0} has var name {1}'
                                 'Coordinate {0} has var name {1} '
                                 'instead of {2}',
                                 coordinate.name,
@@ -509,7 +508,7 @@ def _check_time_coord(self):
         freq = self.frequency
         if freq.lower().endswith('pt'):
             freq = freq[:-2]
-        if freq == 'mon':
+        if freq in ['mon', 'mo']:
             for i in range(len(coord.points) - 1):
                 first = coord.cell(i).point
                 second = coord.cell(i + 1).point

esmvalcore/cmor/table.py

@@ -39,7 +39,11 @@ def read_cmor_tables(cfg_developer):
         cmor_strict = project.get('cmor_strict', True)
         default_table_prefix = project.get('cmor_default_table_prefix', '')
 
-        if cmor_type == 'CMIP5':
+        if cmor_type == 'CMIP3':
+            CMOR_TABLES[table] = CMIP3Info(
+                table_path, default=custom, strict=cmor_strict,
+            )
+        elif cmor_type == 'CMIP5':
             CMOR_TABLES[table] = CMIP5Info(
                 table_path, default=custom, strict=cmor_strict,
             )
@@ -629,7 +633,9 @@ def _read_coordinate(self, value):
             if key in ('variable_entry', 'axis_entry'):
                 return coord
             if key == 'requested':
-                coord.requested = value.split(' ')
+                coord.requested.extend(
+                    (val for val in value.split(' ') if val)
+                )
                 continue
             if hasattr(coord, key):
                 setattr(coord, key, value)
@@ -707,6 +713,46 @@ def get_variable(self, table, short_name, derived=False):
         return var_info
 
 
+class CMIP3Info(CMIP5Info):
+    """
+    Class to read CMIP3-like data request.
+
+    Parameters
+    ----------
+    cmor_tables_path: basestring
+       Path to the folder containing the Tables folder with the json files
+
+    default: object
+        Default table to look variables on if not found
+
+    strict: bool
+        If False, will look for a variable in other tables if it can not be
+        found in the requested one
+
+    """
+
+    def _read_table_file(self, table_file, table=None):
+        for dim in ('zlevel',):
+            coord = CoordinateInfo(dim)
+            coord.generic_level = True
+            coord.axis = 'Z'
+            self.coords[dim] = coord
+        super()._read_table_file(table_file, table)
+
+    def _read_coordinate(self, value):
+        coord = super()._read_coordinate(value)
+        if not coord.out_name:
+            coord.out_name = coord.name
+            coord.var_name = coord.name
+        return coord
+
+    def _read_variable(self, short_name, frequency):
+        var = super()._read_variable(short_name, frequency)
+        var.modeling_realm = None
+        var.frequency = None
+        return var
+
+
 class CustomInfo(CMIP5Info):
     """
     Class to read custom var info for ESMVal.

esmvalcore/cmor/tables/cmip3/.gitignore

@@ -0,0 +1,104 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/

esmvalcore/cmor/tables/cmip3/README.md

@@ -0,0 +1,2 @@
+# cmip3-cmor-tables
+Text Tables for CMOR1 to create a CMIP3 dataset