Skip to content

Commit

Permalink
caching h5 with lat / lon instead of /meta/lat and /meta/lon.
Browse files Browse the repository at this point in the history
  • Loading branch information
bnb32 committed Aug 16, 2024
1 parent e6ed04a commit a8a7e28
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 7 deletions.
2 changes: 0 additions & 2 deletions sup3r/preprocessing/cachers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,8 +225,6 @@ def write_h5(
data_var = data_var.data

dset_name = dset
if dset in Dimension.coords_2d():
dset_name = f'meta/{dset}'
if dset == Dimension.TIME:
dset_name = 'time_index'

Expand Down
2 changes: 1 addition & 1 deletion sup3r/preprocessing/loaders/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ def _add_attrs(self, data):
'source_files': str(self.file_paths),
'date_modified': dt.utcnow().isoformat(),
}
attrs['global_attrs'] = getattr(self.res, 'global_attrs', {})
attrs['global_attrs'] = getattr(self.res, 'global_attrs', ())
attrs.update(getattr(self.res, 'attrs', {}))
data.attrs.update(attrs)
return data
Expand Down
9 changes: 5 additions & 4 deletions sup3r/preprocessing/samplers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,10 +126,11 @@ def preflight(self):
msg = (
f'sample_shape[2] * batch_size ({self.sample_shape[2]} * '
f'{self.batch_size}) is larger than the number of time steps in '
'the raw data. This prevents us from building batches from '
'a single sample with n_time_steps = sample_shape[2] * batch_size '
'which is far more performant than building batches n_samples = '
'batch_size, each with n_time_steps = sample_shape[2].'
'the raw data. This prevents us from building batches with '
'a single sample with n_time_steps = sample_shape[2] * '
'batch_size, which is far more performant than building batches '
'with n_samples = batch_size, each with n_time_steps = '
'sample_shape[2].'
)
if self.data.shape[2] < self.sample_shape[2] * self.batch_size:
logger.warning(msg)
Expand Down

0 comments on commit a8a7e28

Please sign in to comment.