time shift added to solar module, so that daily gcm data can be shift…

…ed to start at the zeroth hour, if it has not been shifted already.

sup3r/preprocessing/names.py

@@ -127,6 +127,7 @@ def dims_4d_bc(cls):
 
 # variables available on a single level (e.g. surface)
 SFC_VARS = [
+    'surface_sensible_heat_flux',
     '10m_u_component_of_wind',
     '10m_v_component_of_wind',
     '100m_u_component_of_wind',

sup3r/solar/solar.py

@@ -35,6 +35,7 @@ def __init__(
         nsrdb_fp,
         t_slice=slice(None),
         tz=-6,
+        time_shift=None,
         agg_factor=1,
         nn_threshold=0.5,
         cloud_threshold=0.99,
@@ -66,6 +67,12 @@ def __init__(
             the GAN is trained on data in local time and therefore the output
             in sup3r_fps should be treated as local time. For example, -6 is
             CST which is default for CONUS training data.
+        time_shift : int | None
+            Number of hours to shift time axis. This can be used, for
+            example, to shift the time index for daily data so that the time
+            stamp for a given day starts at hour zero instead of at
+            noon, as is the case for most GCM data. In this case ``time_shift``
+            would be -12
         agg_factor : int
             Spatial aggregation factor for nsrdb-to-GAN-meta e.g. the number of
             NSRDB spatial pixels to average for a single sup3r GAN output site.
@@ -84,6 +91,7 @@ def __init__(
         self.nn_threshold = nn_threshold
         self.cloud_threshold = cloud_threshold
         self.tz = tz
+        self.time_shift = time_shift
         self._nsrdb_fp = nsrdb_fp
         self._sup3r_fps = sup3r_fps
         if isinstance(self._sup3r_fps, str):
@@ -195,7 +203,8 @@ def time_index(self):
         -------
         pd.DatetimeIndex
         """
-        return self.gan_data.time_index[self.t_slice]
+        ti = self.gan_data.time_index[self.t_slice]
+        return ti.shift(self.time_shift, freq='h')
 
     @property
     def out_of_bounds(self):
@@ -241,7 +250,8 @@ def nsrdb_tslice(self):
                 .mean()
                 .total_seconds()
             )
-            step = int(3600 // delta)
+
+            step = int(3600 / delta)
             self._nsrdb_tslice = slice(t0, t1, step)
 
             logger.debug(
@@ -607,14 +617,6 @@ def run_temporal_chunk(
         fp_out_suffix : str
             Suffix to add to the input sup3r source files when writing the
             processed solar irradiance data to new data files.
-        t_slice : slice
-            Slicing argument to slice the temporal axis of the sup3r_fps source
-            data after doing the tz roll to UTC but before returning the
-            irradiance variables. This can be used to effectively pad the solar
-            irradiance calculation in UTC time. For example, if sup3r_fps is 3
-            files each with 24 hours of data, t_slice can be slice(24, 48) to
-            only output the middle day of irradiance data, but padded by the
-            other two days for the UTC output.
         tz : int
             The timezone offset for the data in sup3r_fps. It is assumed that
             the GAN is trained on data in local time and therefore the output

sup3r/utilities/era_downloader.py

@@ -412,9 +412,17 @@ def _write_dsets(cls, files, out_file, kwargs=None):
             for f in set(ds.data_vars) - set(added_features):
                 mode = 'w' if not os.path.exists(tmp_file) else 'a'
                 logger.info('Adding %s to %s.', f, tmp_file)
-                ds.data[f].load().to_netcdf(
-                    tmp_file, mode=mode, format='NETCDF4', engine='h5netcdf'
-                )
+                try:
+                    ds.data[f].load().to_netcdf(
+                        tmp_file,
+                        mode=mode,
+                        format='NETCDF4',
+                        engine='h5netcdf',
+                    )
+                except Exception as e:
+                    msg = 'Error adding %s from %s to %s. %s'
+                    logger.error(msg, f, file, tmp_file, e)
+                    raise RuntimeError from e
                 logger.info('Added %s to %s.', f, tmp_file)
                 added_features.append(f)
         logger.info(f'Finished writing {tmp_file}')
@@ -550,36 +558,18 @@ def run_month(
         """
         variables = variables if isinstance(variables, list) else [variables]
         for var in variables:
-            try:
-                downloader = cls(
-                    year=year,
-                    month=month,
-                    area=area,
-                    levels=levels,
-                    monthly_file_pattern=monthly_file_pattern,
-                    overwrite=overwrite,
-                    variables=[var],
-                    product_type=product_type,
-                )
-                downloader.get_monthly_file()
-            except Exception as e:
-                msg = (
-                    'Failed to download monthly data for variable = %s, '
-                    'month = %s, year = %s. This was likely an issue with '
-                    'the CDSAPI. Try again later. %s'
-                )
-                logger.warning(msg, var, month, year, e)
-                warn(msg % (var, month, year, e))
-        try:
-            cls.make_monthly_file(year, month, monthly_file_pattern, variables)
-        except Exception as e:
-            msg = (
-                'Could not make monthly file for month = %s, year = %s. '
-                'This was likely caused by a CDSAPI issue when downloading '
-                'some of the variables. %s'
+            downloader = cls(
+                year=year,
+                month=month,
+                area=area,
+                levels=levels,
+                monthly_file_pattern=monthly_file_pattern,
+                overwrite=overwrite,
+                variables=[var],
+                product_type=product_type,
             )
-            logger.warning(msg, month, year, e)
-            warn(msg % (month, year, e))
+            downloader.get_monthly_file()
+        cls.make_monthly_file(year, month, monthly_file_pattern, variables)
 
     @classmethod
     def run(
@@ -662,17 +652,8 @@ def run(
         else:
             dask.compute(*tasks, scheduler='threads', num_workers=max_workers)
 
-        if yearly_file is not None:
-            try:
-                cls.make_yearly_file(year, monthly_file_pattern, yearly_file)
-            except Exception as e:
-                msg = (
-                    'Could not make yearly file for year = %s. This was '
-                    'likely caused by a CDSAPI issue when downloading '
-                    'some of the variables. %s'
-                )
-                logger.warning(msg, year, e)
-                warn(msg % (year, e))
+        if yearly_file is not None and len(months) == 12:
+            cls.make_yearly_file(year, monthly_file_pattern, yearly_file)
 
     @classmethod
     def make_monthly_file(cls, year, month, file_pattern, variables):