diff --git a/include/nc4internal.h b/include/nc4internal.h index 2a1b3558b0..157e6008c7 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -205,6 +205,7 @@ typedef struct NC_VAR_INFO void *fill_value; size_t *chunksizes; nc_bool_t contiguous; /**< True if variable is stored contiguously in HDF5 file */ + nc_bool_t compact; /**< True if variable is in comact storage in HDF5 file */ int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent) */ nc_bool_t dimscale; /**< True if var is a dimscale */ nc_bool_t *dimscale_attached; /**< Array of flags that are true if dimscale is attached for that dim index */ diff --git a/include/netcdf.h b/include/netcdf.h index aa908dd619..cc8370f2b8 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -293,6 +293,7 @@ NOTE: The NC_MAX_DIMS, NC_MAX_ATTRS, and NC_MAX_VARS limits /**@{*/ #define NC_CHUNKED 0 #define NC_CONTIGUOUS 1 +#define NC_COMPACT 2 /**@}*/ /** In HDF5 files you can set check-summing for each variable. diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c index bafa5389cc..8029ad6601 100644 --- a/libdispatch/dvar.c +++ b/libdispatch/dvar.c @@ -452,9 +452,25 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32) /** Define chunking parameters for a variable - The function nc_def_var_chunking sets the chunking parameters for a - variable in a netCDF-4 file. It can set the chunk sizes to get chunked - storage, or it can set the contiguous flag to get contiguous storage. + The function nc_def_var_chunking sets the storage and, optionally, + the chunking parameters for a variable in a netCDF-4 file. + + The storage may be set to NC_CONTIGUOUS, NC_COMPACT, or NC_CHUNKED. + + Contiguous storage means the variable is stored as one block of + data in the file. + + Compact storage means the variable is stored in the header record + of the file. This can have large performance benefits on HPC system + running many processors. Compact storage is only available for + variables whose data are 64 KB or less. Attempting to turn on + compact storage for a variable that is too large will result in the + ::NC_EVARSIZE error. + + Chunked storage means the data are stored as chunks, of + user-configurable size. Chunked storage is required for variable + with one or more unlimted dimensions, or variable which use + compression. The total size of a chunk must be less than 4 GiB. That is, the product of all chunksizes and the size of the data (or the size of @@ -467,20 +483,21 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32) Note that this does not work for scalar variables. Only non-scalar variables can have chunking. - @param[in] ncid NetCDF ID, from a previous call to nc_open or - nc_create. + @param ncid NetCDF ID, from a previous call to nc_open() or + nc_create(). - @param[in] varid Variable ID. + @param varid Variable ID. - @param[in] storage If ::NC_CONTIGUOUS, then contiguous storage is used - for this variable. Variables with one or more unlimited dimensions - cannot use contiguous storage. If contiguous storage is turned on, the - chunksizes parameter is ignored. If ::NC_CHUNKED, then chunked storage - is used for this variable. Chunk sizes may be specified with the - chunksizes parameter or default sizes will be used if that parameter - is NULL. + @param storage If ::NC_CONTIGUOUS or ::NC_COMPACT, then contiguous + or compact storage is used for this variable. Variables with one or + more unlimited dimensions cannot use contiguous or compact + storage. If contiguous or compact storage is turned on, the + chunksizes parameter is ignored. If ::NC_CHUNKED, then chunked + storage is used for this variable. Chunk sizes may be specified + with the chunksizes parameter or default sizes will be used if that + parameter is NULL. - @param[in] chunksizesp A pointer to an array list of chunk sizes. The + @param chunksizesp A pointer to an array list of chunk sizes. The array must have one chunksize for each dimension of the variable. If ::NC_CONTIGUOUS storage is set, then the chunksizes parameter is ignored. @@ -500,6 +517,10 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32) @return ::NC_EBADCHUNK Returns if the chunk size specified for a variable is larger than the length of the dimensions associated with variable. + @return ::NC_EVARSIZE Compact storage attempted for variable bigger + than 64 KB. + @return ::NC_EINVAL Attempt to set contiguous or compact storage + for var with one or more unlimited dimensions. @section nc_def_var_chunking_example Example @@ -539,6 +560,7 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32) if (chunksize[d] != chunksize_in[d]) ERR; if (storage_in != NC_CHUNKED) ERR; @endcode + @author Ed Hartnett, Dennis Heimbigner */ int nc_def_var_chunking(int ncid, int varid, int storage, diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 5d45ef1ebc..52f8b6d5f7 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -1088,8 +1088,10 @@ static int get_chunking_info(hid_t propid, NC_VAR_INFO_T *var) for (d = 0; d < var->ndims; d++) var->chunksizes[d] = chunksize[d]; } - else if (layout == H5D_CONTIGUOUS || layout == H5D_COMPACT) + else if (layout == H5D_CONTIGUOUS) var->contiguous = NC_TRUE; + else if (layout == H5D_COMPACT) + var->compact = NC_TRUE; return NC_NOERR; } diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 78f0cad89e..076238821b 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -22,6 +22,9 @@ * order. */ #define NC_TEMP_NAME "_netcdf4_temporary_variable_name_for_rename" +/** Number of bytes in 64 KB. */ +#define SIXTY_FOUR_KB (65536) + #ifdef LOGGING /** * Report the chunksizes selected for a variable. @@ -707,41 +710,65 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, var->contiguous = NC_FALSE; } - /* Does the user want a contiguous dataset? Not so fast! Make sure - * that there are no unlimited dimensions, and no filters in use - * for this data. */ - if (contiguous && *contiguous) + /* Handle storage settings. */ + if (contiguous) { - if (var->deflate || var->fletcher32 || var->shuffle) - return NC_EINVAL; - - for (d = 0; d < var->ndims; d++) - if (var->dim[d]->unlimited) + /* Does the user want a contiguous or compact dataset? Not so + * fast! Make sure that there are no unlimited dimensions, and + * no filters in use for this data. */ + if (*contiguous) + { + if (var->deflate || var->fletcher32 || var->shuffle) return NC_EINVAL; - var->contiguous = NC_TRUE; - } - /* Chunksizes anyone? */ - if (contiguous && *contiguous == NC_CHUNKED) - { - var->contiguous = NC_FALSE; + for (d = 0; d < var->ndims; d++) + if (var->dim[d]->unlimited) + return NC_EINVAL; + } - /* If the user provided chunksizes, check that they are not too - * big, and that their total size of chunk is less than 4 GB. */ - if (chunksizes) + /* Handle chunked storage settings. */ + if (*contiguous == NC_CHUNKED) { + var->contiguous = NC_FALSE; - if ((retval = check_chunksizes(grp, var, chunksizes))) - return retval; + /* If the user provided chunksizes, check that they are not too + * big, and that their total size of chunk is less than 4 GB. */ + if (chunksizes) + { + /* Check the chunksizes for validity. */ + if ((retval = check_chunksizes(grp, var, chunksizes))) + return retval; - /* Ensure chunksize is smaller than dimension size */ - for (d = 0; d < var->ndims; d++) - if(!var->dim[d]->unlimited && var->dim[d]->len > 0 && chunksizes[d] > var->dim[d]->len) - return NC_EBADCHUNK; + /* Ensure chunksize is smaller than dimension size */ + for (d = 0; d < var->ndims; d++) + if (!var->dim[d]->unlimited && var->dim[d]->len > 0 && + chunksizes[d] > var->dim[d]->len) + return NC_EBADCHUNK; + + /* Set the chunksizes for this variable. */ + for (d = 0; d < var->ndims; d++) + var->chunksizes[d] = chunksizes[d]; + } + } + else if (*contiguous == NC_CONTIGUOUS) + { + var->contiguous = NC_TRUE; + } + else if (*contiguous == NC_COMPACT) + { + size_t ndata = 1; - /* Set the chunksizes for this variable. */ + /* Find the number of elements in the data. */ for (d = 0; d < var->ndims; d++) - var->chunksizes[d] = chunksizes[d]; + ndata *= var->dim[d]->len; + + /* Ensure var is small enough to fit in compact + * storage. It must be <= 64 KB. */ + if (ndata * var->type_info->size > SIXTY_FOUR_KB) + return NC_EVARSIZE; + + var->contiguous = NC_FALSE; + var->compact = NC_TRUE; } } diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index fca9533d4b..7075551dfc 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -994,11 +994,17 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid } } + /* Set the var storage to contiguous, compact, or chunked. */ if (var->contiguous) { if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0) BAIL(NC_EHDFERR); } + else if (var->compact) + { + if (H5Pset_layout(plistid, H5D_COMPACT) < 0) + BAIL(NC_EHDFERR); + } else { if (H5Pset_chunk(plistid, var->ndims, chunksize) < 0) @@ -1106,9 +1112,11 @@ nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) int d; int retval; - /* Nothing to be done. */ - if (var->contiguous) + /* Nothing to be done for contiguous or compact data. */ + if (var->contiguous || var->compact) return NC_NOERR; + + /* No cache adjusting for parallel builds. */ #ifdef USE_PARALLEL4 return NC_NOERR; #endif diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index b953bc8211..de3c5d21ec 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -187,16 +187,26 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, if (nattsp) *nattsp = ncindexcount(var->att); - /* Chunking stuff. */ - if (!var->contiguous && chunksizesp) + /* Did the user want the chunksizes? */ + if (!var->contiguous && !var->compact && chunksizesp) + { for (d = 0; d < var->ndims; d++) { chunksizesp[d] = var->chunksizes[d]; LOG((4, "chunksizesp[%d]=%d", d, chunksizesp[d])); } + } + /* Did the user inquire about the storage? */ if (contiguousp) - *contiguousp = var->contiguous ? NC_CONTIGUOUS : NC_CHUNKED; + { + if (var->contiguous) + *contiguousp = NC_CONTIGUOUS; + else if (var->compact) + *contiguousp = NC_COMPACT; + else + *contiguousp = NC_CHUNKED; + } /* Filter stuff. */ if (deflatep) diff --git a/nc_test4/tst_vars4.c b/nc_test4/tst_vars4.c index 5d8712b428..38c2de2717 100644 --- a/nc_test4/tst_vars4.c +++ b/nc_test4/tst_vars4.c @@ -10,220 +10,268 @@ #include "err_macros.h" #define FILE_NAME "tst_vars4.nc" -#define NDIMS2 2 +#define NDIM2 2 #define NUM_VARS 1 #define Y_NAME "y" #define X_NAME "x" +#define Z_NAME "z" #define VAR_NAME Y_NAME #define XDIM_LEN 2 #define YDIM_LEN 5 +#define ZDIM_LEN 8193 #define CLAIR "Clair" #define JAMIE "Jamie" int main(int argc, char **argv) { - printf("\n*** Testing netcdf-4 variable functions, even more.\n"); - printf("**** testing Jeff's dimension problem..."); - { - int varid, ncid, dims[NDIMS2], dims_in[NDIMS2]; - int ndims, nvars, ngatts, unlimdimid, natts; - char name_in[NC_MAX_NAME + 1]; - nc_type type_in; - size_t len_in; - - if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; - if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR; - if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR; - if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR; - if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; - if (nvars != NUM_VARS || ndims != NDIMS2 || ngatts != 0 || unlimdimid != -1) ERR; - if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR; - if (strcmp(name_in, VAR_NAME) || type_in != NC_FLOAT || ndims != NDIMS2 || - dims_in[0] != dims[0] || dims_in[1] != dims[1] || natts != 0) ERR; - if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; - if (strcmp(name_in, X_NAME) || len_in != XDIM_LEN) ERR; - if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR; - if (strcmp(name_in, Y_NAME)) ERR; - if (len_in != YDIM_LEN) ERR; - if (nc_close(ncid)) ERR; - - /* Open the file and check. */ - if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; - if (nvars != NUM_VARS || ndims != NDIMS2 || ngatts != 0 || unlimdimid != -1) ERR; - if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR; - if (strcmp(name_in, VAR_NAME) || type_in != NC_FLOAT || ndims != NDIMS2 || - dims_in[0] != dims[0] || dims_in[1] != dims[1] || natts != 0) ERR; - if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; - if (strcmp(name_in, X_NAME) || len_in != XDIM_LEN) ERR; - if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR; - if (strcmp(name_in, Y_NAME)) ERR; - if (len_in != YDIM_LEN) ERR; - if (nc_close(ncid)) ERR; - } - SUMMARIZE_ERR; - printf("**** testing chunking turned on by fletcher..."); - { - int varid, ncid, dims[NDIMS2]; - int storage_in; - size_t chunksizes_in[NDIMS2]; - - if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; - if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR; - if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR; - if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR; - if (nc_def_var_fletcher32(ncid, varid, NC_FLETCHER32)) ERR; - if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR; - if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR; - if (nc_close(ncid)) ERR; - - /* Open the file and check. */ - if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR; - if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR; - if (nc_close(ncid)) ERR; - } - SUMMARIZE_ERR; - printf("**** testing chunking turned on by shuffle..."); - { - int varid, ncid, dims[NDIMS2]; - int storage_in; - size_t chunksizes_in[NDIMS2]; - - if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; - if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR; - if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR; - if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR; - if (nc_def_var_deflate(ncid, varid, NC_SHUFFLE, 0, 0)) ERR; - if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR; - if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR; - if (nc_close(ncid)) ERR; - - /* Open the file and check. */ - if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR; - if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR; - if (nc_close(ncid)) ERR; - } - SUMMARIZE_ERR; + printf("\n*** Testing netcdf-4 variable functions, even more.\n"); + printf("**** testing Jeff's dimension problem..."); + { + int varid, ncid, dims[NDIM2], dims_in[NDIM2]; + int ndims, nvars, ngatts, unlimdimid, natts; + char name_in[NC_MAX_NAME + 1]; + nc_type type_in; + size_t len_in; + + if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR; + if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR; + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR; + if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; + if (nvars != NUM_VARS || ndims != NDIM2 || ngatts != 0 || unlimdimid != -1) ERR; + if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR; + if (strcmp(name_in, VAR_NAME) || type_in != NC_FLOAT || ndims != NDIM2 || + dims_in[0] != dims[0] || dims_in[1] != dims[1] || natts != 0) ERR; + if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; + if (strcmp(name_in, X_NAME) || len_in != XDIM_LEN) ERR; + if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR; + if (strcmp(name_in, Y_NAME)) ERR; + if (len_in != YDIM_LEN) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR; + if (nvars != NUM_VARS || ndims != NDIM2 || ngatts != 0 || unlimdimid != -1) ERR; + if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR; + if (strcmp(name_in, VAR_NAME) || type_in != NC_FLOAT || ndims != NDIM2 || + dims_in[0] != dims[0] || dims_in[1] != dims[1] || natts != 0) ERR; + if (nc_inq_dim(ncid, 0, name_in, &len_in)) ERR; + if (strcmp(name_in, X_NAME) || len_in != XDIM_LEN) ERR; + if (nc_inq_dim(ncid, 1, name_in, &len_in)) ERR; + if (strcmp(name_in, Y_NAME)) ERR; + if (len_in != YDIM_LEN) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing chunking turned on by fletcher..."); + { + int varid, ncid, dims[NDIM2]; + int storage_in; + size_t chunksizes_in[NDIM2]; + + if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR; + if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR; + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR; + if (nc_def_var_fletcher32(ncid, varid, NC_FLETCHER32)) ERR; + if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR; + if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR; + if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing chunking turned on by shuffle..."); + { + int varid, ncid, dims[NDIM2]; + int storage_in; + size_t chunksizes_in[NDIM2]; + + if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR; + if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR; + if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR; + if (nc_def_var_deflate(ncid, varid, NC_SHUFFLE, 0, 0)) ERR; + if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR; + if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR; + if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; #define DIM_NAME "Distance_from_Mayo" #define VAR_NAME_2 "Rocky_Road_to_Dublin" #define NDIMS1 1 #define NUM_RECORDS 3 - printf("**** testing extending var along unlimited dim with no coord var..."); - { - int varid, ncid, dimid; - int ndims, nvars, natts, unlimdimid; - size_t dim_len_in, index; - int data = TEST_VAL_42; - - /* Create the test file with one var, one unlimited dim. */ - if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; - if (nc_def_dim(ncid, DIM_NAME, NC_UNLIMITED, &dimid)) ERR; - if (nc_def_var(ncid, VAR_NAME_2, NC_INT, NDIMS1, &dimid, &varid)) ERR; - - /* Write some records. */ - for (index = 0; index < NUM_RECORDS; index++) - if (nc_put_var1_int(ncid, varid, &index, &data)) ERR; - if (nc_close(ncid)) ERR; - - /* Open the file and check. */ - if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; - if (ndims != 1 || nvars != 1 || natts != 0 || unlimdimid != 0) ERR; - if (nc_inq_dim(ncid, dimid, NULL, &dim_len_in)) ERR; - if (dim_len_in != NUM_RECORDS) ERR; - - /* Now add more records. */ - for (index = 3; index < NUM_RECORDS * 2; index++) - if (nc_put_var1_int(ncid, varid, &index, &data)) ERR; - if (nc_inq_dim(ncid, dimid, NULL, &dim_len_in)) ERR; - - if (dim_len_in != NUM_RECORDS * 2) ERR; - - /* Close the file. */ - if (nc_close(ncid)) ERR; - } - SUMMARIZE_ERR; - printf("**** testing type creation and destruction for atomic types..."); - { - int varid1, varid2, ncid; - int ndims, nvars, natts, unlimdimid; - - /* Create the test file with two scalar vars. */ - /* nc_set_log_level(4); */ - if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; - if (nc_def_var(ncid, CLAIR, NC_INT, 0, NULL, &varid1)) ERR; - if (nc_def_var(ncid, JAMIE, NC_INT, 0, NULL, &varid2)) ERR; - if (nc_close(ncid)) ERR; - - /* Open the file and check. */ - if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; - if (ndims != 0 || nvars != 2 || natts != 0 || unlimdimid != -1) ERR; - if (nc_close(ncid)) ERR; - } - SUMMARIZE_ERR; - printf("**** testing scalar big endian vars..."); - { - int varid1, varid2, ncid; - int ndims, nvars, natts, unlimdimid; - int test_val = TEST_VAL_42; - int test_val2 = TEST_VAL_42 * 2; - int data_in; - - /* Create the test file with two scalar vars. */ - if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; - if (nc_def_var(ncid, CLAIR, NC_INT, 0, NULL, &varid1)) ERR; - if (nc_def_var_endian(ncid, varid1, NC_ENDIAN_BIG)) ERR; - if (nc_def_var(ncid, JAMIE, NC_INT, 0, NULL, &varid2)) ERR; - if (nc_def_var_endian(ncid, varid2, NC_ENDIAN_BIG)) ERR; - if (nc_enddef(ncid)) ERR; - if (nc_put_var(ncid, varid1, &test_val)) ERR; - if (nc_put_var(ncid, varid2, &test_val2)) ERR; - if (nc_close(ncid)) ERR; - - /* Open the file and check. */ - if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; - if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; - if (ndims != 0 || nvars != 2 || natts != 0 || unlimdimid != -1) ERR; - if (nc_get_var(ncid, varid1, &data_in)) ERR; - if (data_in != TEST_VAL_42) ERR; - if (nc_get_var(ncid, varid2, &data_in)) ERR; - if (data_in != TEST_VAL_42 * 2) ERR; - if (nc_close(ncid)) ERR; - } - SUMMARIZE_ERR; - printf("**** testing scalar big endian vars..."); - { - int ncid, enumid; - int bigid, littleid; - int endian_in; - /* Note: if no zero valued enum, then causes ncdump error */ - int econst0 = 0; - int econst1 = 1; - - if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR; - - if (nc_def_enum(ncid, NC_INT, "enum_t", &enumid)) ERR; - if (nc_insert_enum(ncid, enumid, "econst0", &econst0)) ERR; - if (nc_insert_enum(ncid, enumid, "econst1", &econst1)) ERR; - - if (nc_def_var(ncid, "little", enumid, 0, NULL, &littleid)) ERR; - if (nc_def_var(ncid, "big", enumid, 0, NULL, &bigid)) ERR; - - if (nc_def_var_endian(ncid, littleid, NC_ENDIAN_LITTLE) != NC_EINVAL) ERR; - if (nc_def_var_endian(ncid, bigid, NC_ENDIAN_BIG) != NC_EINVAL) ERR; - - /* Note that it is important to set endian ness before testing it */ - if (nc_inq_var_endian(ncid, littleid, &endian_in)) ERR; - if (endian_in) ERR; - if (nc_inq_var_endian(ncid, bigid, &endian_in)) ERR; - if (endian_in) ERR; - - if (nc_close(ncid)) ERR; - } - SUMMARIZE_ERR; - FINAL_RESULTS; + printf("**** testing extending var along unlimited dim with no coord var..."); + { + int varid, ncid, dimid; + int ndims, nvars, natts, unlimdimid; + size_t dim_len_in, index; + int data = TEST_VAL_42; + + /* Create the test file with one var, one unlimited dim. */ + if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME, NC_UNLIMITED, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME_2, NC_INT, NDIMS1, &dimid, &varid)) ERR; + + /* Write some records. */ + for (index = 0; index < NUM_RECORDS; index++) + if (nc_put_var1_int(ncid, varid, &index, &data)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; + if (ndims != 1 || nvars != 1 || natts != 0 || unlimdimid != 0) ERR; + if (nc_inq_dim(ncid, dimid, NULL, &dim_len_in)) ERR; + if (dim_len_in != NUM_RECORDS) ERR; + + /* Now add more records. */ + for (index = 3; index < NUM_RECORDS * 2; index++) + if (nc_put_var1_int(ncid, varid, &index, &data)) ERR; + if (nc_inq_dim(ncid, dimid, NULL, &dim_len_in)) ERR; + + if (dim_len_in != NUM_RECORDS * 2) ERR; + + /* Close the file. */ + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing type creation and destruction for atomic types..."); + { + int varid1, varid2, ncid; + int ndims, nvars, natts, unlimdimid; + + /* Create the test file with two scalar vars. */ + /* nc_set_log_level(4); */ + if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + if (nc_def_var(ncid, CLAIR, NC_INT, 0, NULL, &varid1)) ERR; + if (nc_def_var(ncid, JAMIE, NC_INT, 0, NULL, &varid2)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; + if (ndims != 0 || nvars != 2 || natts != 0 || unlimdimid != -1) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing scalar big endian vars..."); + { + int varid1, varid2, ncid; + int ndims, nvars, natts, unlimdimid; + int test_val = TEST_VAL_42; + int test_val2 = TEST_VAL_42 * 2; + int data_in; + + /* Create the test file with two scalar vars. */ + if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; + if (nc_def_var(ncid, CLAIR, NC_INT, 0, NULL, &varid1)) ERR; + if (nc_def_var_endian(ncid, varid1, NC_ENDIAN_BIG)) ERR; + if (nc_def_var(ncid, JAMIE, NC_INT, 0, NULL, &varid2)) ERR; + if (nc_def_var_endian(ncid, varid2, NC_ENDIAN_BIG)) ERR; + if (nc_enddef(ncid)) ERR; + if (nc_put_var(ncid, varid1, &test_val)) ERR; + if (nc_put_var(ncid, varid2, &test_val2)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file and check. */ + if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; + if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; + if (ndims != 0 || nvars != 2 || natts != 0 || unlimdimid != -1) ERR; + if (nc_get_var(ncid, varid1, &data_in)) ERR; + if (data_in != TEST_VAL_42) ERR; + if (nc_get_var(ncid, varid2, &data_in)) ERR; + if (data_in != TEST_VAL_42 * 2) ERR; + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing scalar big endian vars..."); + { + int ncid, enumid; + int bigid, littleid; + int endian_in; + /* Note: if no zero valued enum, then causes ncdump error */ + int econst0 = 0; + int econst1 = 1; + + if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR; + + if (nc_def_enum(ncid, NC_INT, "enum_t", &enumid)) ERR; + if (nc_insert_enum(ncid, enumid, "econst0", &econst0)) ERR; + if (nc_insert_enum(ncid, enumid, "econst1", &econst1)) ERR; + + if (nc_def_var(ncid, "little", enumid, 0, NULL, &littleid)) ERR; + if (nc_def_var(ncid, "big", enumid, 0, NULL, &bigid)) ERR; + + if (nc_def_var_endian(ncid, littleid, NC_ENDIAN_LITTLE) != NC_EINVAL) ERR; + if (nc_def_var_endian(ncid, bigid, NC_ENDIAN_BIG) != NC_EINVAL) ERR; + + /* Note that it is important to set endian ness before testing it */ + if (nc_inq_var_endian(ncid, littleid, &endian_in)) ERR; + if (endian_in) ERR; + if (nc_inq_var_endian(ncid, bigid, &endian_in)) ERR; + if (endian_in) ERR; + + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; + printf("**** testing compact storage..."); + { + int ncid, dimid[NDIM2], varid, varid2; + int data[XDIM_LEN]; + int x; + + /* Create some data. */ + for (x = 0; x < XDIM_LEN; x++) + data[x] = x; + + /* Create a file with one var with compact storage. */ + if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR; + + /* Define dims. */ + if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dimid[0])) ERR; + if (nc_def_dim(ncid, Z_NAME, ZDIM_LEN, &dimid[1])) ERR; + + /* Define vars. */ + if (nc_def_var(ncid, Y_NAME, NC_INT, 1, dimid, &varid)) ERR; + if (nc_def_var_chunking(ncid, varid, NC_COMPACT, NULL)) ERR; + if (nc_def_var(ncid, CLAIR, NC_INT, NDIM2, dimid, &varid2)) ERR; + /* This won't work, the var is too big for compact! */ + if (nc_def_var_chunking(ncid, varid2, NC_COMPACT, NULL) != NC_EVARSIZE) ERR; + + /* Write data. */ + if (nc_put_var_int(ncid, varid, data)) ERR; + + /* Close file. */ + if (nc_close(ncid)) ERR; + + /* Open the file and check it. */ + { + int ndims, nvars; + int storage_in; + + if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR; + if (nc_inq(ncid, &ndims, &nvars, NULL, NULL)) ERR; + if (ndims != 2 || nvars != 2) ERR; + if (nc_inq_var_chunking(ncid, varid, &storage_in, NULL)) ERR; + if (storage_in != NC_COMPACT) ERR; + if (nc_inq_var_chunking(ncid, varid2, &storage_in, NULL)) ERR; + if (storage_in != NC_CONTIGUOUS) ERR; + if (nc_close(ncid)) ERR; + } + } + SUMMARIZE_ERR; + FINAL_RESULTS; }