Unidata · WardF · Jul 6, 2020 · Jul 2, 2020 · Jul 2, 2020 · Jul 2, 2020
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.8.0 - TBD
 
+* [Bug Fix] Now allow szip to be used on variables with unlimited dimension [https://github.com/Unidata/netcdf-c/issues/1774].
 * [Enhancement] Add support for cloud storage using a variant of the Zarr storage format. Warning: this feature is highly experimental and is subject to rapid evolution [https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in].
 * [Bug Fix] Fix nccopy to properly set default chunking parameters when not otherwise specified. This can significantly improve performance in selected cases. Note that if seeing slow performance with nccopy, then, as a work-around, specifically set the chunking parameters. [https://github.com/Unidata/netcdf-c/issues/1763].
 * [Bug Fix] Fix some protocol bugs/differences between the netcdf-c library and the OPeNDAP Hyrax server. Also cleanup checksum handling [https://github.com/Unidata/netcdf-c/issues/1712].

diff --git a/libhdf5/hdf5filter.c b/libhdf5/hdf5filter.c
@@ -317,7 +317,8 @@ NC4_filter_actions(int ncid, int varid, int op, void* args)
             size_t num_elem = 1;
             int d;
             for (d = 0; d < var->ndims; d++)
-                num_elem *= var->dim[d]->len;
+                if (var->dim[d]->len)
+		    num_elem *= var->dim[d]->len;
             /* Pixels per block must be <= number of elements. */
             if (params[1] > num_elem)
                 {stat = THROW(NC_EINVAL); goto done;}

diff --git a/nc_test4/tst_parallel_compress.c b/nc_test4/tst_parallel_compress.c
@@ -18,7 +18,7 @@
 #include "err_macros.h"
 #include <mpi.h>
 
-#define FILE_NAME "tst_parallel_zlib2.nc"
+#define FILE_NAME "tst_parallel_compress.nc"
 #define NDIMS 3
 #define DIMSIZE 24
 #define QTR_DATA (DIMSIZE * DIMSIZE / 4)
@@ -43,7 +43,7 @@ main(int argc, char **argv)
     int ncid, v1id, dimids[NDIMS];
     size_t start[NDIMS], count[NDIMS];
 
-    int f, i, res;
+    int f, i, s, res;
     int *slab_data; /* one slab */
 
     /* Initialize MPI. */
@@ -62,7 +62,6 @@ main(int argc, char **argv)
     if (!mpi_rank)
        printf("\n*** Testing parallel writes with compression filters.\n");
     {
-        int s;
         for (f = 0; f < NUM_COMPRESSION_FILTERS; f++)
         {
             for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++)
@@ -177,6 +176,123 @@ main(int argc, char **argv)
                     SUMMARIZE_ERR;
             } /* next shuffle filter test */
         } /* next compression filter (zlib and szip) */
+
+	/* Now run tests with unlimited dim. */
+        /* for (f = 0; f < NUM_COMPRESSION_FILTERS; f++) */
+        for (f = 1; f < NUM_COMPRESSION_FILTERS; f++)
+        {
+            for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++)
+            {
+                if (!mpi_rank)
+                {
+                    printf("*** testing write along unlim dim with %s shuffle %d...",
+                           (f ? "szip" : "zlib"), s);
+                }
+
+                /* nc_set_log_level(3); */
+                /* Create a parallel netcdf-4 file. */
+                if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR;
+
+                /* Create three dimensions. */
+                if (nc_def_dim(ncid, "d1", DIMSIZE, &dimids[1])) ERR;
+                if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[2])) ERR;
+                if (nc_def_dim(ncid, "d3", NC_UNLIMITED, &dimids[0])) ERR;
+
+                /* Create one var. Turn on deflation. */
+                if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
+
+                /* Setting any filter only will work for HDF5-1.10.3 and later
+                 * versions. */
+                if (!f)
+                    res = nc_def_var_deflate(ncid, 0, s, 1, 1);
+                else
+                {
+                    res = nc_def_var_deflate(ncid, 0, s, 0, 0);
+                    if (!res)
+                        res = nc_def_var_szip(ncid, 0, 32, 32);
+                }
+#ifdef HDF5_SUPPORTS_PAR_FILTERS
+                if (res) ERR;
+#else
+                if (res != NC_EINVAL) ERR;
+#endif
+
+                /* Setting fletcher32 only will work for HDF5-1.10.3 and later
+                 * versions. */
+                res = nc_def_var_fletcher32(ncid, 0, 1);
+#ifdef HDF5_SUPPORTS_PAR_FILTERS
+                if (res) ERR;
+#else
+                if (res != NC_EINVAL) ERR;
+#endif
+
+                /* Write metadata to file. */
+                if (nc_enddef(ncid)) ERR;
+
+                /* Set up slab for this process. */
+                start[1] = mpi_rank * DIMSIZE/mpi_size;
+                start[2] = 0;
+                count[1] = DIMSIZE/mpi_size;
+                count[2] = DIMSIZE;
+                count[0] = 1;
+                /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
+                  mpi_rank, start[0], start[1], count[0], count[1]);*/
+
+                /* Should not be allowed to change access to independent,
+                 * because filters are in use. */
+                if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT) != NC_EINVAL) ERR;
+
+                /* Write slabs of data. */
+                for (start[0] = 0; start[0] < NUM_SLABS; start[0]++)
+                    if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
+
+                /* Close the netcdf file. */
+                if (nc_close(ncid)) ERR;
+
+                /* Check file. */
+                {
+                    int shuffle_in, deflate_in, deflate_level_in;
+                    int options_mask_in, pixels_per_block_in;
+                    int *slab_data_in;
+
+                    /* Allocate data. */
+                    if (!(slab_data_in = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR;
+
+                    /* Reopen the file for parallel access. */
+                    if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR;
+
+                    /* Check state of compression. */
+                    if (!f)
+                    {
+                        if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR;
+                        if ((s && !shuffle_in) || (!s && shuffle_in)) ERR;
+                        if (!deflate_in || deflate_level_in != 1) ERR;
+                    }
+                    else
+                    {
+                        if (nc_inq_var_deflate(ncid, 0, &shuffle_in, NULL, NULL)) ERR;
+                        if ((s && !shuffle_in) || (!s && shuffle_in)) ERR;
+                        if (nc_inq_var_szip(ncid, 0, &options_mask_in, &pixels_per_block_in)) ERR;
+                    }
+
+                    /* Use parallel I/O to read the data. */
+                    for (start[0] = 0; start[0] < NUM_SLABS; start[0]++)
+                    {
+                        if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR;
+                        for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++)
+                            if (slab_data_in[i] != mpi_rank) ERR;
+                    }
+
+                    /* Close the netcdf file. */
+                    if (nc_close(ncid)) ERR;
+
+                    free(slab_data_in);
+                }
+
+                if (!mpi_rank)
+                    SUMMARIZE_ERR;
+            } /* next shuffle filter test */
+        } /* next compression filter (zlib and szip) */
         free(slab_data);
     }
 

diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am
@@ -132,30 +132,30 @@ ref_tst_noncoord.cdl ref_tst_compounds2.nc ref_tst_compounds2.cdl	\
 ref_tst_compounds3.nc ref_tst_compounds3.cdl ref_tst_compounds4.nc	\
 ref_tst_compounds4.cdl ref_tst_group_data_v23.cdl tst_mslp.cdl		\
 tst_bug321.cdl ref_tst_format_att.cdl ref_tst_format_att_64.cdl		\
-tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh ref_nc_test_netcdf4_4_0.nc		\
-run_back_comp_tests.sh ref_nc_test_netcdf4.cdl				\
-ref_tst_special_atts3.cdl tst_brecs.cdl ref_tst_grp_spec0.cdl		\
-ref_tst_grp_spec.cdl tst_grp_spec.sh ref_tst_charfill.cdl		\
-tst_charfill.cdl tst_charfill.sh tst_iter.sh tst_mud.sh			\
-ref_tst_mud4.cdl ref_tst_mud4-bc.cdl ref_tst_mud4_chars.cdl		\
-inttags.cdl inttags4.cdl ref_inttags.cdl ref_inttags4.cdl		\
-ref_tst_ncf213.cdl tst_h_scalar.sh run_utf8_nc4_tests.sh		\
-tst_formatx3.sh tst_formatx4.sh ref_tst_utf8_4.cdl			\
-ref_tst_nc4_utf8_4.cdl tst_inttags.sh tst_inttags4.sh CMakeLists.txt	\
-tst_bom.sh tst_inmemory_nc3.sh tst_dimsizes.sh		\
-tst_inmemory_nc4.sh tst_fileinfo.sh run_ncgen_tests.sh			\
-ref_test_360_day_1900.nc ref_test_365_day_1900.nc			\
+tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh				\
+ref_nc_test_netcdf4_4_0.nc run_back_comp_tests.sh			\
+ref_nc_test_netcdf4.cdl ref_tst_special_atts3.cdl tst_brecs.cdl		\
+ref_tst_grp_spec0.cdl ref_tst_grp_spec.cdl tst_grp_spec.sh		\
+ref_tst_charfill.cdl tst_charfill.cdl tst_charfill.sh tst_iter.sh	\
+tst_mud.sh ref_tst_mud4.cdl ref_tst_mud4-bc.cdl				\
+ref_tst_mud4_chars.cdl inttags.cdl inttags4.cdl ref_inttags.cdl		\
+ref_inttags4.cdl ref_tst_ncf213.cdl tst_h_scalar.sh			\
+run_utf8_nc4_tests.sh tst_formatx3.sh tst_formatx4.sh			\
+ref_tst_utf8_4.cdl ref_tst_nc4_utf8_4.cdl tst_inttags.sh		\
+tst_inttags4.sh CMakeLists.txt tst_bom.sh tst_inmemory_nc3.sh		\
+tst_dimsizes.sh tst_inmemory_nc4.sh tst_fileinfo.sh			\
+run_ncgen_tests.sh ref_test_360_day_1900.nc ref_test_365_day_1900.nc	\
 ref_test_366_day_1900.nc ref_test_360_day_1900.cdl			\
 ref_test_365_day_1900.cdl ref_test_366_day_1900.cdl			\
 tst_hdf5_offset.sh run_ncgen_nc4_tests.sh tst_nccopy3_subset.sh		\
 ref_nccopy3_subset.nc ref_test_corrupt_magic.nc tst_ncgen_shared.sh	\
 tst_ncgen4.sh tst_ncgen4_classic.sh tst_ncgen4_diff.sh			\
 tst_ncgen4_cycle.sh tst_null_byte_padding.sh				\
-ref_null_byte_padding_test.nc ref_tst_irish_rover.nc ref_provenance_v1.nc \
-ref_tst_radix.cdl tst_radix.cdl test_radix.sh                           \
-ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh ref_no_ncproperty.nc \
-test_unicode_directory.sh
-
+ref_null_byte_padding_test.nc ref_tst_irish_rover.nc			\
+ref_provenance_v1.nc ref_tst_radix.cdl tst_radix.cdl test_radix.sh	\
+ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh			\
+ref_no_ncproperty.nc test_unicode_directory.sh				\
+ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl
 
 # The L512.bin file is file containing exactly 512 bytes each of value 0.
 # It is used for creating hdf5 files with varying offsets for testing.
@@ -188,6 +188,7 @@ compound_datasize_test.nc compound_datasize_test2.nc ncf199.nc		\
 tst_c0.cdl tst_c0_4.cdl tst_c0_4c.cdl tst_c0_64.cdl			\
 tst_compound_datasize_test.cdl tst_compound_datasize_test2.cdl		\
 tst_ncf199.cdl tst_tst_gattenum.cdl tst_tst_usuffix.cdl ctest.c		\
-ctest64.c nccopy3_subset_out.nc camrun.c tst_ncf213.cdl tst_ncf213.nc   \
-tst_radix.nc tmp_radix.cdl ctest_small_3.c ctest_small_4.c              \
-ctest_special_atts_4.c
+ctest64.c nccopy3_subset_out.nc camrun.c tst_ncf213.cdl tst_ncf213.nc	\
+tst_radix.nc tmp_radix.cdl ctest_small_3.c ctest_small_4.c		\
+ctest_special_atts_4.c tst_roman_szip_simple.cdl			\
+tst_roman_szip_unlim.cdl
diff --git a/ncdump/ref_roman_szip_simple.cdl b/ncdump/ref_roman_szip_simple.cdl
@@ -0,0 +1,14 @@
+netcdf tst_roman_szip_simple {
+dimensions:
+	Centuria = 100 ;
+variables:
+	int Legio_tertia_Gallica(Centuria) ;
+data:
+
+ Legio_tertia_Gallica = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 
+    16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 
+    34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 
+    52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 
+    70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 
+    88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 ;
+}
diff --git a/ncdump/ref_roman_szip_unlim.cdl b/ncdump/ref_roman_szip_unlim.cdl
@@ -0,0 +1,21 @@
+netcdf tst_roman_szip_unlim {
+dimensions:
+	Primi_ordinis = UNLIMITED ; // (1 currently)
+	Centuria = 10 ;
+	heredia = 10 ;
+variables:
+	int Legio_tertia_Gallica(Primi_ordinis, Centuria, heredia) ;
+data:
+
+ Legio_tertia_Gallica =
+  0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
+  10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
+  20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
+  30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
+  40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
+  50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
+  60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
+  70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
+  80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
+  90, 91, 92, 93, 94, 95, 96, 97, 98, 99 ;
+}
diff --git a/ncdump/tst_create_files.c b/ncdump/tst_create_files.c
@@ -5,7 +5,7 @@
    This program creates some test files which ncdump will read. This
    is only done if netCDF-4 is enabled.
 
-   $Id: tst_create_files.c,v 1.16 2008/10/20 01:48:08 ed Exp $
+   Ed Hartnett 2008/10/20
 */
 
 #include <nc_tests.h>
@@ -263,5 +263,60 @@ main(int argc, char **argv)
 
    SUMMARIZE_ERR;
 
+   /* These files only get created for builds in which szip support is
+    * present in HDF5. */
+#ifdef HAVE_H5Z_SZIP
+#define SZIP_DIM_NAME "Centuria"
+#define SZIP_DIM_LEN 100
+#define SZIP_VAR_NAME "Legio_tertia_Gallica"
+#define FILE_NAME_SZIP_SIMPLE "tst_roman_szip_simple.nc"
+   printf("*** creating simple file with szip compression %s...", FILE_NAME_SZIP_SIMPLE);
+   {
+       int ncid, dimid, varid;
+       int data[SZIP_DIM_LEN];
+       int i;
+
+       for (i = 0; i < SZIP_DIM_LEN; i++)
+	   data[i] = i;
+
+      /* Create a file with szip compression. */
+      if (nc_create(FILE_NAME_SZIP_SIMPLE, NC_NETCDF4, &ncid)) ERR;
+      if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN, &dimid)) ERR;
+      if (nc_def_var(ncid, SZIP_VAR_NAME, NC_INT, 1, &dimid, &varid)) ERR;
+      if (nc_def_var_szip(ncid, varid, 32, 32)) ERR;
+      if (nc_put_var(ncid, varid, data)) ERR;
+
+      if (nc_close(ncid)) ERR;
+   }
+   SUMMARIZE_ERR;
+#define NDIM3 3
+#define SZIP_OTHER_DIM_NAME "heredia"
+#define SZIP_UNLIM_DIM_NAME "Primi_ordinis"
+#define FILE_NAME_SZIP_UNLIM "tst_roman_szip_unlim.nc"
+#define SZIP_DIM_LEN_10 10
+   printf("*** creating file with szip compression and unlim dim %s...", FILE_NAME_SZIP_UNLIM);
+   {
+       int ncid, dimid[NDIM3], varid;
+       int data[SZIP_DIM_LEN_10 * SZIP_DIM_LEN_10];
+       size_t start[NDIM3] = {0, 0, 0};
+       size_t count[NDIM3] = {1, SZIP_DIM_LEN_10, SZIP_DIM_LEN_10};
+       int i;
+
+       for (i = 0; i < SZIP_DIM_LEN_10 * SZIP_DIM_LEN_10; i++)
+	   data[i] = i;
+
+      /* Create a file with szip compression. */
+      if (nc_create(FILE_NAME_SZIP_UNLIM, NC_NETCDF4, &ncid)) ERR;
+      if (nc_def_dim(ncid, SZIP_UNLIM_DIM_NAME, NC_UNLIMITED, &dimid[0])) ERR;
+      if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN_10, &dimid[1])) ERR;
+      if (nc_def_dim(ncid, SZIP_OTHER_DIM_NAME, SZIP_DIM_LEN_10, &dimid[2])) ERR;
+      if (nc_def_var(ncid, SZIP_VAR_NAME, NC_INT, 3, dimid, &varid)) ERR;
+      if (nc_def_var_szip(ncid, varid, 32, 32)) ERR;
+      if (nc_put_vara_int(ncid, varid, start, count, data)) ERR;
+
+      if (nc_close(ncid)) ERR;
+   }
+   SUMMARIZE_ERR;
+#endif /* HAVE_H5Z_SZIP */
    FINAL_RESULTS;
 }
diff --git a/ncdump/tst_netcdf4.sh b/ncdump/tst_netcdf4.sh
@@ -49,6 +49,14 @@ diff -b tst_solar_1.cdl $srcdir/ref_tst_solar_1.cdl ; ERR
 ${NCDUMP} tst_solar_2.nc | sed 's/e+0/e+/g' > tst_solar_2.cdl ; ERR
 diff -b tst_solar_2.cdl $srcdir/ref_tst_solar_2.cdl ; ERR
 
+if test -f tst_roman_szip_simple.nc; then
+  echo "*** Testing szip compression."    
+  ${NCDUMP} tst_roman_szip_simple.nc | sed 's/e+0/e+/g' > tst_roman_szip_simple.cdl ; ERR
+  diff -b tst_roman_szip_simple.cdl $srcdir/ref_roman_szip_simple.cdl ; ERR
+  ${NCDUMP} tst_roman_szip_unlim.nc | sed 's/e+0/e+/g' > tst_roman_szip_unlim.cdl ; ERR
+  diff -b tst_roman_szip_unlim.cdl $srcdir/ref_roman_szip_unlim.cdl ; ERR
+fi
+
 echo "*** Running tst_group_data.c to create test files."
 ${execdir}/tst_group_data ; ERR
 ${NCDUMP} tst_group_data.nc | sed 's/e+0/e+/g' > tst_group_data.cdl ; ERR