diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 2115e2456f..48a8b2fcb4 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.0 - TBD +* [Bug Fix] Now allow szip to be used on variables with unlimited dimension [https://github.com/Unidata/netcdf-c/issues/1774]. * [Enhancement] Add support for cloud storage using a variant of the Zarr storage format. Warning: this feature is highly experimental and is subject to rapid evolution [https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in]. * [Bug Fix] Fix nccopy to properly set default chunking parameters when not otherwise specified. This can significantly improve performance in selected cases. Note that if seeing slow performance with nccopy, then, as a work-around, specifically set the chunking parameters. [https://github.com/Unidata/netcdf-c/issues/1763]. * [Bug Fix] Fix some protocol bugs/differences between the netcdf-c library and the OPeNDAP Hyrax server. Also cleanup checksum handling [https://github.com/Unidata/netcdf-c/issues/1712]. diff --git a/libhdf5/hdf5filter.c b/libhdf5/hdf5filter.c index 2c5c7fda18..c118bd23f4 100644 --- a/libhdf5/hdf5filter.c +++ b/libhdf5/hdf5filter.c @@ -317,7 +317,8 @@ NC4_filter_actions(int ncid, int varid, int op, void* args) size_t num_elem = 1; int d; for (d = 0; d < var->ndims; d++) - num_elem *= var->dim[d]->len; + if (var->dim[d]->len) + num_elem *= var->dim[d]->len; /* Pixels per block must be <= number of elements. */ if (params[1] > num_elem) {stat = THROW(NC_EINVAL); goto done;} diff --git a/nc_test4/tst_parallel_compress.c b/nc_test4/tst_parallel_compress.c index a621a47637..f8823feb5c 100644 --- a/nc_test4/tst_parallel_compress.c +++ b/nc_test4/tst_parallel_compress.c @@ -18,7 +18,7 @@ #include "err_macros.h" #include -#define FILE_NAME "tst_parallel_zlib2.nc" +#define FILE_NAME "tst_parallel_compress.nc" #define NDIMS 3 #define DIMSIZE 24 #define QTR_DATA (DIMSIZE * DIMSIZE / 4) @@ -43,7 +43,7 @@ main(int argc, char **argv) int ncid, v1id, dimids[NDIMS]; size_t start[NDIMS], count[NDIMS]; - int f, i, res; + int f, i, s, res; int *slab_data; /* one slab */ /* Initialize MPI. */ @@ -62,7 +62,6 @@ main(int argc, char **argv) if (!mpi_rank) printf("\n*** Testing parallel writes with compression filters.\n"); { - int s; for (f = 0; f < NUM_COMPRESSION_FILTERS; f++) { for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++) @@ -177,6 +176,123 @@ main(int argc, char **argv) SUMMARIZE_ERR; } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ + + /* Now run tests with unlimited dim. */ + /* for (f = 0; f < NUM_COMPRESSION_FILTERS; f++) */ + for (f = 1; f < NUM_COMPRESSION_FILTERS; f++) + { + for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++) + { + if (!mpi_rank) + { + printf("*** testing write along unlim dim with %s shuffle %d...", + (f ? "szip" : "zlib"), s); + } + + /* nc_set_log_level(3); */ + /* Create a parallel netcdf-4 file. */ + if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR; + + /* Create three dimensions. */ + if (nc_def_dim(ncid, "d1", DIMSIZE, &dimids[1])) ERR; + if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[2])) ERR; + if (nc_def_dim(ncid, "d3", NC_UNLIMITED, &dimids[0])) ERR; + + /* Create one var. Turn on deflation. */ + if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR; + + /* Setting any filter only will work for HDF5-1.10.3 and later + * versions. */ + if (!f) + res = nc_def_var_deflate(ncid, 0, s, 1, 1); + else + { + res = nc_def_var_deflate(ncid, 0, s, 0, 0); + if (!res) + res = nc_def_var_szip(ncid, 0, 32, 32); + } +#ifdef HDF5_SUPPORTS_PAR_FILTERS + if (res) ERR; +#else + if (res != NC_EINVAL) ERR; +#endif + + /* Setting fletcher32 only will work for HDF5-1.10.3 and later + * versions. */ + res = nc_def_var_fletcher32(ncid, 0, 1); +#ifdef HDF5_SUPPORTS_PAR_FILTERS + if (res) ERR; +#else + if (res != NC_EINVAL) ERR; +#endif + + /* Write metadata to file. */ + if (nc_enddef(ncid)) ERR; + + /* Set up slab for this process. */ + start[1] = mpi_rank * DIMSIZE/mpi_size; + start[2] = 0; + count[1] = DIMSIZE/mpi_size; + count[2] = DIMSIZE; + count[0] = 1; + /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n", + mpi_rank, start[0], start[1], count[0], count[1]);*/ + + /* Should not be allowed to change access to independent, + * because filters are in use. */ + if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT) != NC_EINVAL) ERR; + + /* Write slabs of data. */ + for (start[0] = 0; start[0] < NUM_SLABS; start[0]++) + if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR; + + /* Close the netcdf file. */ + if (nc_close(ncid)) ERR; + + /* Check file. */ + { + int shuffle_in, deflate_in, deflate_level_in; + int options_mask_in, pixels_per_block_in; + int *slab_data_in; + + /* Allocate data. */ + if (!(slab_data_in = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR; + + /* Reopen the file for parallel access. */ + if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR; + + /* Check state of compression. */ + if (!f) + { + if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR; + if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; + if (!deflate_in || deflate_level_in != 1) ERR; + } + else + { + if (nc_inq_var_deflate(ncid, 0, &shuffle_in, NULL, NULL)) ERR; + if ((s && !shuffle_in) || (!s && shuffle_in)) ERR; + if (nc_inq_var_szip(ncid, 0, &options_mask_in, &pixels_per_block_in)) ERR; + } + + /* Use parallel I/O to read the data. */ + for (start[0] = 0; start[0] < NUM_SLABS; start[0]++) + { + if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR; + for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++) + if (slab_data_in[i] != mpi_rank) ERR; + } + + /* Close the netcdf file. */ + if (nc_close(ncid)) ERR; + + free(slab_data_in); + } + + if (!mpi_rank) + SUMMARIZE_ERR; + } /* next shuffle filter test */ + } /* next compression filter (zlib and szip) */ free(slab_data); } diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 9da3bf0439..9fc057987c 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -132,30 +132,30 @@ ref_tst_noncoord.cdl ref_tst_compounds2.nc ref_tst_compounds2.cdl \ ref_tst_compounds3.nc ref_tst_compounds3.cdl ref_tst_compounds4.nc \ ref_tst_compounds4.cdl ref_tst_group_data_v23.cdl tst_mslp.cdl \ tst_bug321.cdl ref_tst_format_att.cdl ref_tst_format_att_64.cdl \ -tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh ref_nc_test_netcdf4_4_0.nc \ -run_back_comp_tests.sh ref_nc_test_netcdf4.cdl \ -ref_tst_special_atts3.cdl tst_brecs.cdl ref_tst_grp_spec0.cdl \ -ref_tst_grp_spec.cdl tst_grp_spec.sh ref_tst_charfill.cdl \ -tst_charfill.cdl tst_charfill.sh tst_iter.sh tst_mud.sh \ -ref_tst_mud4.cdl ref_tst_mud4-bc.cdl ref_tst_mud4_chars.cdl \ -inttags.cdl inttags4.cdl ref_inttags.cdl ref_inttags4.cdl \ -ref_tst_ncf213.cdl tst_h_scalar.sh run_utf8_nc4_tests.sh \ -tst_formatx3.sh tst_formatx4.sh ref_tst_utf8_4.cdl \ -ref_tst_nc4_utf8_4.cdl tst_inttags.sh tst_inttags4.sh CMakeLists.txt \ -tst_bom.sh tst_inmemory_nc3.sh tst_dimsizes.sh \ -tst_inmemory_nc4.sh tst_fileinfo.sh run_ncgen_tests.sh \ -ref_test_360_day_1900.nc ref_test_365_day_1900.nc \ +tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh \ +ref_nc_test_netcdf4_4_0.nc run_back_comp_tests.sh \ +ref_nc_test_netcdf4.cdl ref_tst_special_atts3.cdl tst_brecs.cdl \ +ref_tst_grp_spec0.cdl ref_tst_grp_spec.cdl tst_grp_spec.sh \ +ref_tst_charfill.cdl tst_charfill.cdl tst_charfill.sh tst_iter.sh \ +tst_mud.sh ref_tst_mud4.cdl ref_tst_mud4-bc.cdl \ +ref_tst_mud4_chars.cdl inttags.cdl inttags4.cdl ref_inttags.cdl \ +ref_inttags4.cdl ref_tst_ncf213.cdl tst_h_scalar.sh \ +run_utf8_nc4_tests.sh tst_formatx3.sh tst_formatx4.sh \ +ref_tst_utf8_4.cdl ref_tst_nc4_utf8_4.cdl tst_inttags.sh \ +tst_inttags4.sh CMakeLists.txt tst_bom.sh tst_inmemory_nc3.sh \ +tst_dimsizes.sh tst_inmemory_nc4.sh tst_fileinfo.sh \ +run_ncgen_tests.sh ref_test_360_day_1900.nc ref_test_365_day_1900.nc \ ref_test_366_day_1900.nc ref_test_360_day_1900.cdl \ ref_test_365_day_1900.cdl ref_test_366_day_1900.cdl \ tst_hdf5_offset.sh run_ncgen_nc4_tests.sh tst_nccopy3_subset.sh \ ref_nccopy3_subset.nc ref_test_corrupt_magic.nc tst_ncgen_shared.sh \ tst_ncgen4.sh tst_ncgen4_classic.sh tst_ncgen4_diff.sh \ tst_ncgen4_cycle.sh tst_null_byte_padding.sh \ -ref_null_byte_padding_test.nc ref_tst_irish_rover.nc ref_provenance_v1.nc \ -ref_tst_radix.cdl tst_radix.cdl test_radix.sh \ -ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh ref_no_ncproperty.nc \ -test_unicode_directory.sh - +ref_null_byte_padding_test.nc ref_tst_irish_rover.nc \ +ref_provenance_v1.nc ref_tst_radix.cdl tst_radix.cdl test_radix.sh \ +ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh \ +ref_no_ncproperty.nc test_unicode_directory.sh \ +ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl # The L512.bin file is file containing exactly 512 bytes each of value 0. # It is used for creating hdf5 files with varying offsets for testing. @@ -188,6 +188,7 @@ compound_datasize_test.nc compound_datasize_test2.nc ncf199.nc \ tst_c0.cdl tst_c0_4.cdl tst_c0_4c.cdl tst_c0_64.cdl \ tst_compound_datasize_test.cdl tst_compound_datasize_test2.cdl \ tst_ncf199.cdl tst_tst_gattenum.cdl tst_tst_usuffix.cdl ctest.c \ -ctest64.c nccopy3_subset_out.nc camrun.c tst_ncf213.cdl tst_ncf213.nc \ -tst_radix.nc tmp_radix.cdl ctest_small_3.c ctest_small_4.c \ -ctest_special_atts_4.c +ctest64.c nccopy3_subset_out.nc camrun.c tst_ncf213.cdl tst_ncf213.nc \ +tst_radix.nc tmp_radix.cdl ctest_small_3.c ctest_small_4.c \ +ctest_special_atts_4.c tst_roman_szip_simple.cdl \ +tst_roman_szip_unlim.cdl diff --git a/ncdump/ref_roman_szip_simple.cdl b/ncdump/ref_roman_szip_simple.cdl new file mode 100644 index 0000000000..07bf9416c3 --- /dev/null +++ b/ncdump/ref_roman_szip_simple.cdl @@ -0,0 +1,14 @@ +netcdf tst_roman_szip_simple { +dimensions: + Centuria = 100 ; +variables: + int Legio_tertia_Gallica(Centuria) ; +data: + + Legio_tertia_Gallica = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 ; +} diff --git a/ncdump/ref_roman_szip_unlim.cdl b/ncdump/ref_roman_szip_unlim.cdl new file mode 100644 index 0000000000..9c019d49c6 --- /dev/null +++ b/ncdump/ref_roman_szip_unlim.cdl @@ -0,0 +1,21 @@ +netcdf tst_roman_szip_unlim { +dimensions: + Primi_ordinis = UNLIMITED ; // (1 currently) + Centuria = 10 ; + heredia = 10 ; +variables: + int Legio_tertia_Gallica(Primi_ordinis, Centuria, heredia) ; +data: + + Legio_tertia_Gallica = + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 ; +} diff --git a/ncdump/tst_create_files.c b/ncdump/tst_create_files.c index be9d55662b..c8122042a9 100644 --- a/ncdump/tst_create_files.c +++ b/ncdump/tst_create_files.c @@ -5,7 +5,7 @@ This program creates some test files which ncdump will read. This is only done if netCDF-4 is enabled. - $Id: tst_create_files.c,v 1.16 2008/10/20 01:48:08 ed Exp $ + Ed Hartnett 2008/10/20 */ #include @@ -263,5 +263,60 @@ main(int argc, char **argv) SUMMARIZE_ERR; + /* These files only get created for builds in which szip support is + * present in HDF5. */ +#ifdef HAVE_H5Z_SZIP +#define SZIP_DIM_NAME "Centuria" +#define SZIP_DIM_LEN 100 +#define SZIP_VAR_NAME "Legio_tertia_Gallica" +#define FILE_NAME_SZIP_SIMPLE "tst_roman_szip_simple.nc" + printf("*** creating simple file with szip compression %s...", FILE_NAME_SZIP_SIMPLE); + { + int ncid, dimid, varid; + int data[SZIP_DIM_LEN]; + int i; + + for (i = 0; i < SZIP_DIM_LEN; i++) + data[i] = i; + + /* Create a file with szip compression. */ + if (nc_create(FILE_NAME_SZIP_SIMPLE, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN, &dimid)) ERR; + if (nc_def_var(ncid, SZIP_VAR_NAME, NC_INT, 1, &dimid, &varid)) ERR; + if (nc_def_var_szip(ncid, varid, 32, 32)) ERR; + if (nc_put_var(ncid, varid, data)) ERR; + + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; +#define NDIM3 3 +#define SZIP_OTHER_DIM_NAME "heredia" +#define SZIP_UNLIM_DIM_NAME "Primi_ordinis" +#define FILE_NAME_SZIP_UNLIM "tst_roman_szip_unlim.nc" +#define SZIP_DIM_LEN_10 10 + printf("*** creating file with szip compression and unlim dim %s...", FILE_NAME_SZIP_UNLIM); + { + int ncid, dimid[NDIM3], varid; + int data[SZIP_DIM_LEN_10 * SZIP_DIM_LEN_10]; + size_t start[NDIM3] = {0, 0, 0}; + size_t count[NDIM3] = {1, SZIP_DIM_LEN_10, SZIP_DIM_LEN_10}; + int i; + + for (i = 0; i < SZIP_DIM_LEN_10 * SZIP_DIM_LEN_10; i++) + data[i] = i; + + /* Create a file with szip compression. */ + if (nc_create(FILE_NAME_SZIP_UNLIM, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, SZIP_UNLIM_DIM_NAME, NC_UNLIMITED, &dimid[0])) ERR; + if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN_10, &dimid[1])) ERR; + if (nc_def_dim(ncid, SZIP_OTHER_DIM_NAME, SZIP_DIM_LEN_10, &dimid[2])) ERR; + if (nc_def_var(ncid, SZIP_VAR_NAME, NC_INT, 3, dimid, &varid)) ERR; + if (nc_def_var_szip(ncid, varid, 32, 32)) ERR; + if (nc_put_vara_int(ncid, varid, start, count, data)) ERR; + + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; +#endif /* HAVE_H5Z_SZIP */ FINAL_RESULTS; } diff --git a/ncdump/tst_netcdf4.sh b/ncdump/tst_netcdf4.sh index 5f7b98c4d5..d3737e3592 100755 --- a/ncdump/tst_netcdf4.sh +++ b/ncdump/tst_netcdf4.sh @@ -49,6 +49,14 @@ diff -b tst_solar_1.cdl $srcdir/ref_tst_solar_1.cdl ; ERR ${NCDUMP} tst_solar_2.nc | sed 's/e+0/e+/g' > tst_solar_2.cdl ; ERR diff -b tst_solar_2.cdl $srcdir/ref_tst_solar_2.cdl ; ERR +if test -f tst_roman_szip_simple.nc; then + echo "*** Testing szip compression." + ${NCDUMP} tst_roman_szip_simple.nc | sed 's/e+0/e+/g' > tst_roman_szip_simple.cdl ; ERR + diff -b tst_roman_szip_simple.cdl $srcdir/ref_roman_szip_simple.cdl ; ERR + ${NCDUMP} tst_roman_szip_unlim.nc | sed 's/e+0/e+/g' > tst_roman_szip_unlim.cdl ; ERR + diff -b tst_roman_szip_unlim.cdl $srcdir/ref_roman_szip_unlim.cdl ; ERR +fi + echo "*** Running tst_group_data.c to create test files." ${execdir}/tst_group_data ; ERR ${NCDUMP} tst_group_data.nc | sed 's/e+0/e+/g' > tst_group_data.cdl ; ERR