Skip to content

Commit

Permalink
Merge pull request #751 from PCMDI/725_expose_netcdf_quantize_and_zst…
Browse files Browse the repository at this point in the history 
…andard

Expose NetCDF quantize and zstandard functions
  • Loading branch information
@mauzey1
mauzey1 authored Aug 27, 2024
2 parents c3be34f + 81e46f4 commit e1852d0
Show file tree
Hide file tree
Showing 14 changed files with 477 additions and 20 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/nightly-build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,9 @@ jobs:
export DST_META_YAML=`pwd`/$PROJECT_DIR/cmor-feedstock/recipe/meta.yaml
mv $DST_META_YAML $SRC_META_YAML
export GIT_REV=$(git rev-parse --short HEAD)
export GIT_REV=$(git rev-parse --short "$GITHUB_SHA")
echo "GIT_REV=$GIT_REV"
export GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
export GIT_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
echo "GIT_BRANCH=$GIT_BRANCH"
python rebuild_meta_yaml.py \
Expand Down
5 changes: 3 additions & 2 deletions Lib/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@
close, grid, set_grid_mapping, time_varying_grid_coordinate, dataset_json,
set_cur_dataset_attribute, get_cur_dataset_attribute,
has_cur_dataset_attribute, set_variable_attribute, get_variable_attribute,
has_variable_attribute, get_final_filename, set_deflate, set_furtherinfourl,
set_climatology, get_climatology, set_terminate_signal, get_terminate_signal)
has_variable_attribute, get_final_filename, set_deflate, set_zstandard,
set_quantize, set_furtherinfourl, set_climatology, get_climatology,
set_terminate_signal, get_terminate_signal)

try:
from check_CMOR_compliant import checkCMOR
Expand Down
35 changes: 35 additions & 0 deletions Lib/pywrapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1032,6 +1032,41 @@ def set_deflate(var_id, shuffle, deflate, deflate_level):
return _cmor.set_deflate(var_id, shuffle, deflate, deflate_level)


def set_zstandard(var_id, zstandar_level):
"""Sets Zstandard compression on a cmor variable
Usage:
cmor.set_zstandard(var_id, szstandar_level)
Where:
var_id: is cmor variable id
zstandar_level: Compression level. Must be set from -131072 to 22
"""

return _cmor.set_zstandard(var_id, zstandar_level)


def set_quantize(var_id, quantize_mode, quantize_nsd):
"""Sets quantization on a cmor variable
Usage:
cmor.set_quantize(var_id, quantize_mode, quantize_nsd)
Where:
var_id: is cmor variable id
quantize_mode: Quantization mode. Can be set to the following.
0: No quantization mode
1: BitGroom
2: Granular BitRound
3: BitRound
quantize_nsd: Number of significant digits. If quantize_mode is set to
1 or 2, then the value can be set from 1 to 7 for floats
and 1 to 23 for doubles. If quantize_mode is set to 3, then
the value can be set from 1 to 15 for floats and 1 to 52
for doubles. The value is ignore if quantize_mode is 0.
"""

return _cmor.set_quantize(var_id, quantize_mode, quantize_nsd)


def has_variable_attribute(var_id, name):
"""determines if the a cmor variable has an attribute
Usage:
Expand Down
1 change: 1 addition & 0 deletions Makefile.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,7 @@ test_python: python
env TEST_NAME=Test/test_cmor_python_not_enough_times_written.py make test_a_python
env TEST_NAME=Test/test_python_forecast_coordinates.py make test_a_python
env TEST_NAME=Test/test_cmor_CMIP6Plus.py make test_a_python
env TEST_NAME=Test/test_cmor_zstandard_and_quantize.py make test_a_python
test_cmip6_cv: python
env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentnotset.py make test_a_python
env TEST_NAME=Test/test_python_CMIP6_CV_sub_experimentbad.py make test_a_python
Expand Down
48 changes: 48 additions & 0 deletions Src/_cmormodule.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,52 @@ static PyObject *PyCMOR_set_deflate(PyObject * self, PyObject * args)
return (Py_BuildValue("i", ierr));
}

/************************************************************************/
/* PyCMOR_set_zstandard() */
/************************************************************************/
static PyObject *PyCMOR_set_zstandard(PyObject * self, PyObject * args)
{
signal(signal_to_catch, signal_handler);
int ierr, var_id, zstandard_level;

if (!PyArg_ParseTuple
(args, "ii", &var_id, &zstandard_level))
return NULL;

ierr = cmor_set_zstandard(var_id, zstandard_level);

if (ierr != 0 || raise_exception) {
raise_exception = 0;
PyErr_Format(CMORError, exception_message, "set_zstandard");
return NULL;
}

return (Py_BuildValue("i", ierr));
}

/************************************************************************/
/* PyCMOR_set_quantize() */
/************************************************************************/
static PyObject *PyCMOR_set_quantize(PyObject * self, PyObject * args)
{
signal(signal_to_catch, signal_handler);
int ierr, var_id, quantize_mode, quantize_nsd;

if (!PyArg_ParseTuple
(args, "iii", &var_id, &quantize_mode, &quantize_nsd))
return NULL;

ierr = cmor_set_quantize(var_id, quantize_mode, quantize_nsd);

if (ierr != 0 || raise_exception) {
raise_exception = 0;
PyErr_Format(CMORError, exception_message, "set_quantize");
return NULL;
}

return (Py_BuildValue("i", ierr));
}

/************************************************************************/
/* PyCMOR_set_variable_attribute() */
/************************************************************************/
Expand Down Expand Up @@ -1146,6 +1192,8 @@ static PyMethodDef MyExtractMethods[] = {
{"set_furtherinfourl", PyCMOR_set_furtherinfourl, METH_VARARGS},
{"get_final_filename", PyCMOR_getFinalFilename, METH_VARARGS},
{"set_deflate", PyCMOR_set_deflate, METH_VARARGS},
{"set_zstandard", PyCMOR_set_zstandard, METH_VARARGS},
{"set_quantize", PyCMOR_set_quantize, METH_VARARGS},
{"set_terminate_signal", PyCMOR_set_terminate_signal, METH_VARARGS},
{"get_terminate_signal", PyCMOR_get_terminate_signal, METH_VARARGS},
{NULL, NULL} /*sentinel */
Expand Down
103 changes: 90 additions & 13 deletions Src/cmor.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include "cmor.h"
#include "cmor_locale.h"
#include <netcdf.h>
#include <netcdf_filter.h>
#include <udunits2.h>
#include <time.h>
#include <errno.h>
Expand Down Expand Up @@ -41,6 +42,20 @@ int nc_def_var_chunking(int i, int j, int k, size_t * l)
};
#endif

#ifndef H5Z_FILTER_ZSTD
int nc_def_var_zstandard(int i, int j, int k)
{
return (0);
};
#endif

#ifndef NC_QUANTIZE_BITGROOM
int nc_def_var_quantize(int i, int j, int k, int l)
{
return (0);
};
#endif

/* -------------------------------------------------------------------- */
/* function declaration */
/* -------------------------------------------------------------------- */
Expand Down Expand Up @@ -720,6 +735,9 @@ void cmor_reset_variable(int var_id)
cmor_vars[var_id].shuffle = 0;
cmor_vars[var_id].deflate = 1;
cmor_vars[var_id].deflate_level = 1;
cmor_vars[var_id].zstandard_level = 3;
cmor_vars[var_id].quantize_mode = 0;
cmor_vars[var_id].quantize_nsd = 1;
cmor_vars[var_id].nomissing = 1;
cmor_vars[var_id].iunits[0] = '\0';
cmor_vars[var_id].ounits[0] = '\0';
Expand Down Expand Up @@ -1869,7 +1887,7 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim,
int ierr = 0, l, m, k, n, j, m2, found, nelts, *int_list = NULL;
int dim_holder[CMOR_MAX_VARIABLES];
int lnzfactors;
int ics, icd, icdl, ia;
int ics, icd, icdl, icz, icqm, icqn, ia;
cmor_add_traceback("cmor_define_zfactors_vars");
cmor_is_setup();
lnzfactors = *nzfactors;
Expand Down Expand Up @@ -2066,8 +2084,24 @@ int cmor_define_zfactors_vars(int var_id, int ncid, int *nc_dim,
icd = cmor_tables[nTableID].vars[nTableID].deflate;
icdl =
cmor_tables[nTableID].vars[nTableID].deflate_level;
ierr = nc_def_var_deflate(ncid, nc_zfactors[lnzfactors],
ics, icd, icdl);
icz =
cmor_tables[nTableID].vars[nTableID].zstandard_level;
icqm =
cmor_tables[nTableID].vars[nTableID].quantize_mode;
icqn =
cmor_tables[nTableID].vars[nTableID].quantize_nsd;

ierr = nc_def_var_quantize(ncid, nc_zfactors[lnzfactors],
icqm, icqn);
if (icd != 0) {
ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors],
ics, icd, icdl);
} else {
ierr |= nc_def_var_deflate(ncid, nc_zfactors[lnzfactors],
ics, 0, 0);
ierr |= nc_def_var_zstandard(ncid, nc_zfactors[lnzfactors],
icz);
}

if (ierr != NC_NOERR) {
snprintf(msg, CMOR_MAX_STRING,
Expand Down Expand Up @@ -3432,7 +3466,7 @@ void cmor_define_dimensions(int var_id, int ncid,
int tmp_dims[2];
int dims_bnds_ids[2];
int nVarRefTblID = cmor_vars[var_id].ref_table_id;
int ics, icd, icdl;
int ics, icd, icdl, icz, icqm, icqn;
int itmpmsg, itmp2, itmp3;
int maxStrLen;

Expand Down Expand Up @@ -3784,9 +3818,21 @@ void cmor_define_dimensions(int var_id, int ncid,
ics = pVar->shuffle;
icd = pVar->deflate;
icdl = pVar->deflate_level;

ierr = nc_def_var_deflate(ncafid, nc_bnds_vars[i], ics, icd,
icdl);
icz = pVar->zstandard_level;
icqm = pVar->quantize_mode;
icqn = pVar->quantize_nsd;

ierr = nc_def_var_quantize(ncafid, nc_bnds_vars[i], icqm,
icqn);
if (icd != 0) {
ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i],
ics, icd, icdl);
} else {
ierr |= nc_def_var_deflate(ncafid, nc_bnds_vars[i],
ics, 0, 0);
ierr |= nc_def_var_zstandard(ncafid, nc_bnds_vars[i],
icz);
}
if (ierr != NC_NOERR) {
snprintf(msg, CMOR_MAX_STRING,
"NCError (%i: %s) defining compression\n! "
Expand Down Expand Up @@ -4039,7 +4085,7 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af,
int *int_list = NULL;
char mtype;
int nelts;
int ics, icd, icdl;
int ics, icd, icdl, icz, icqm, icqn;

cmor_add_traceback("cmor_grids_def");
/* -------------------------------------------------------------------- */
Expand Down Expand Up @@ -4306,9 +4352,29 @@ int cmor_grids_def(int var_id, int nGridID, int ncafid, int *nc_dim_af,
cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j].
ref_var_id].
deflate_level;

ierr = nc_def_var_deflate(ncafid, nc_associated_vars[i],
ics, icd, icdl);
icz =
cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j].
ref_var_id].
zstandard_level;
icqm =
cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j].
ref_var_id].
quantize_mode;
icqn =
cmor_tables[cmor_vars[j].ref_table_id].vars[cmor_vars[j].
ref_var_id].
quantize_nsd;
ierr = nc_def_var_quantize(ncafid, nc_associated_vars[i],
icqm, icqn);
if (icd != 0) {
ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i],
ics, icd, icdl);
} else {
ierr |= nc_def_var_deflate(ncafid, nc_associated_vars[i],
ics, 0, 0);
ierr |= nc_def_var_zstandard(ncafid, nc_associated_vars[i],
icz);
}
if (ierr != NC_NOERR) {
snprintf(msg, CMOR_MAX_STRING,
"NetCDF Error (%i: %s) defining\n! "
Expand Down Expand Up @@ -5093,7 +5159,7 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid,
int nVarRefTblID = cmor_vars[var_id].ref_table_id;
int nelts;
int *int_list = NULL;
int ics, icd, icdl;
int ics, icd, icdl, icz, icqm, icqn;
int bChunk;
cmor_add_traceback("cmor_create_var_attributes");
/* -------------------------------------------------------------------- */
Expand Down Expand Up @@ -5167,7 +5233,18 @@ void cmor_create_var_attributes(int var_id, int ncid, int ncafid,
ics = pVar->shuffle;
icd = pVar->deflate;
icdl = pVar->deflate_level;
ierr = nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl);
icz = pVar->zstandard_level;
icqm = pVar->quantize_mode;
icqn = pVar->quantize_nsd;
ierr = nc_def_var_quantize(ncid, pVar->nc_var_id, icqm, icqn);

// Only use zstandard compression if deflate is disabled
if (icd != 0) {
ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, icd, icdl);
} else {
ierr |= nc_def_var_deflate(ncid, pVar->nc_var_id, ics, 0, 0);
ierr |= nc_def_var_zstandard(ncid, pVar->nc_var_id, icz);
}

if (ierr != NC_NOERR) {
snprintf(msg, CMOR_MAX_STRING,
Expand Down
3 changes: 3 additions & 0 deletions Src/cmor_CV.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2539,6 +2539,9 @@ int cmor_CV_variable(int *var_id, char *name, char *units,
cmor_vars[vrid].shuffle = refvar.shuffle;
cmor_vars[vrid].deflate = refvar.deflate;
cmor_vars[vrid].deflate_level = refvar.deflate_level;
cmor_vars[vrid].zstandard_level = refvar.zstandard_level;
cmor_vars[vrid].quantize_mode = refvar.quantize_mode;
cmor_vars[vrid].quantize_nsd = refvar.quantize_nsd;
cmor_vars[vrid].first_bound = startimebnds;
cmor_vars[vrid].last_bound = endtimebnds;
cmor_vars[vrid].first_time = startime;
Expand Down
17 changes: 17 additions & 0 deletions Src/cmor_cfortran_interface.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,23 @@ int cmor_set_deflate_cff_(int *var_id, int *shuffle,
return (cmor_set_deflate(*var_id, *shuffle, *deflate, *deflate_level));
}

/************************************************************************/
/* cmor_set_zstandard_cff_() */
/************************************************************************/
int cmor_set_zstandard_cff_(int *var_id, int *zstandard_level)
{
return (cmor_set_zstandard(*var_id, *zstandard_level));
}

/************************************************************************/
/* cmor_set_quantize_cff_() */
/************************************************************************/
int cmor_set_quantize_cff_(int *var_id, int *quantize_mode,
int *quantize_level)
{
return (cmor_set_quantize(*var_id, *quantize_mode, *quantize_level));
}

/************************************************************************/
/* cmor_get_variable_attribute_cff_() */
/************************************************************************/
Expand Down
Loading

0 comments on commit e1852d0

Please sign in to comment.