Extend nccopy -F option syntax.

debug/cf

@@ -3,7 +3,7 @@
 DB=1
 #X=-x
 
-ANSI=1
+#ANSI=1
 #MEM=1
 #NOTUIL=1
 #FAST=1

docs/filters.md

@@ -138,23 +138,37 @@ The "-F" option can be used repeatedly as long as the variable name
 part is different. A different filter id and parameters can be
 specified for each occurrence.
 
+It can be convenient to specify that the same compression is to be
+applied to more than one variable. To support this, two additional
+*-F* cases are defined.
+
+1. ````-F *,...``` means apply the filter to all variables in the dataset.
+2. ````-F v1|v2|..,...``` means apply the filter to a multiple variables.
+
+Note that the characters '*' and '|' are bash reserved characters,
+so you will probably need to escape or quote the filter spec in
+that environment.
+
 As a rule, any input filter on an input variable will be applied
 to the equivalent output variable -- assuming the output file type
 is netcdf-4. It is, however, sometimes convenient to suppress
 output compression either totally or on a per-variable basis.
 Total suppression of output filters can be accomplished by specifying
 a special case of "-F", namely this.
 ````
-nccopy -F "none" input.nc output.nc
+nccopy -F none input.nc output.nc
 ````
-Suppression of output filtering for a specific variable can be accomplished
-using this format.
+The expression ````-F *,none```` is equivalent to ````-F none````.
+
+Suppression of output filtering for a specific set of variables
+can be accomplished using these formats.
 ````
 nccopy -F "var,none" input.nc output.nc
+nccopy -F "v1|v2|...,none" input.nc output.nc
 ````
-where "var" is the fully qualified name of the variable.
+where "var" and the "vi" are the fully qualified name of a variable.
 
-The rules for all possible cases of the "-F" flag are defined
+The rules for all possible cases of the "-F none" flag are defined
 by this table.
 
 <table>
@@ -165,6 +179,7 @@ by this table.
 <tr><td>false<td>unspecified<td>defined<td>use input filter
 <tr><td>false<td>-Fvar,none<td>NA<td>unfiltered
 <tr><td>false<td>-Fvar,...<td>NA<td>use output filter
+<tr><td>false<td>unspecified<td>none<td>unfiltered
 </table> 
 
 # Parameter Encode/Decode {#filters_paramcoding}

nc_test4/Makefile.am

@@ -135,8 +135,8 @@ run_grp_rename.sh tst_h5_endians.c tst_atts_string_rewrite.c		\
 tst_put_vars_two_unlim_dim.c tst_empty_vlen_unlim.c			\
 run_empty_vlen_test.sh ref_hdf5_compat1.nc ref_hdf5_compat2.nc		\
 ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5	\
-ref_szip.cdl tst_filter.sh bzip2.cdl filtered.cdl unfiltered.cdl	\
-ref_bzip2.c findplugin.in perftest.sh
+ref_szip.cdl tst_filter.sh bzip2.cdl ref_filtered.cdl ref_unfiltered.cdl \
+ref_bzip2.c findplugin.in perftest.sh ref_unfilteredvv.cdl ref_filteredvv.cdl
 
 CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc	\
 tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc			\

nc_test4/ref_filteredvv.cdl

@@ -0,0 +1,41 @@
+netcdf filteredvv {
+dimensions:
+	dim0 = 4 ;
+	dim1 = 4 ;
+variables:
+	float var1(dim0, dim1) ;
+		var1:_Storage = "chunked" ;
+		var1:_ChunkSizes = 2, 2 ;
+		var1:_Endianness = "little" ;
+		var1:_Filter = "307,9,4" ;
+		var1:_NoFill = "true" ;
+
+// global attributes:
+		:_Format = "netCDF-4" ;
+data:
+
+ var1 =
+  100, 101, 102, 103,
+  104, 105, 106, 107,
+  108, 109, 1010, 1011,
+  1012, 1013, 1014, 1015 ;
+
+group: g {
+  variables:
+  	float var2(dim0, dim1) ;
+  		var2:_Storage = "chunked" ;
+  		var2:_ChunkSizes = 2, 2 ;
+  		var2:_Endianness = "little" ;
+  		var2:_Filter = "307,9,4" ;
+  		var2:_NoFill = "true" ;
+
+  // group attributes:
+  data:
+
+   var2 =
+  0, 1, 2, 3,
+  4, 5, 6, 7,
+  8, 9, 10, 11,
+  12, 13, 14, 15 ;
+  } // group g
+}

nc_test4/ref_unfilteredvv.cdl

@@ -0,0 +1,29 @@
+netcdf unfilteredvv {
+dimensions:
+	dim0 = 4 ;
+	dim1 = 4 ;
+variables:
+	float var1(dim0, dim1) ;
+ 	    var1:_ChunkSizes = 2, 2 ;
+data:
+
+ var1 =
+  100, 101, 102, 103,
+  104, 105, 106, 107,
+  108, 109, 1010, 1011,
+  1012, 1013, 1014, 1015 ;
+
+group: g {
+  variables:
+  	float var2(dim0, dim1) ;
+ 	    var2:_ChunkSizes = 2, 2 ;
+
+  data:
+
+   var2 =
+  0, 1, 2, 3,
+  4, 5, 6, 7,
+  8, 9, 10, 11,
+  12, 13, 14, 15 ;
+  } // group g
+}

nc_test4/tst_filter.sh

@@ -30,7 +30,12 @@ cat $1 \
 # Function to extract _Filter attribute from a file
 # These attributes might be platform dependent
 getfilterattr() {
-sed -e '/var:_Filter/p' -ed <$1 >$2
+case "$1" in
+var1) sed -e '/var1:_Filter/p' -ed <$1 >$2 ;;
+var2) sed -e '/var2:_Filter/p' -ed <$1 >$2 ;;
+var) sed -e '/var:_Filter/p' -ed <$1 >$2 ;;
+*) sed -e '/var:_Filter/p' -ed <$1 >$2 ;;
+esac
 }
 
 trimleft() {
@@ -98,15 +103,34 @@ fi
 if test "x$NCP" = x1 ; then
 echo "*** Testing dynamic filters using nccopy"
 rm -f ./unfiltered.nc ./filtered.nc ./tmp.nc ./filtered.dump ./tst_filter.txt
-${NCGEN} -4 -lb -o unfiltered.nc ${srcdir}/unfiltered.cdl
+# Create our input test files
+${NCGEN} -4 -lb -o unfiltered.nc ${srcdir}/ref_unfiltered.cdl
+${NCGEN} -4 -lb -o unfilteredvv.nc ${srcdir}/ref_unfilteredvv.cdl
+
 echo "	*** Testing simple filter application"
 ${NCCOPY} -M0 -F "/g/var,307,9,4" unfiltered.nc filtered.nc
 ${NCDUMP} -s filtered.nc > ./tst_filter.txt
 # Remove irrelevant -s output
 sclean ./tst_filter.txt ./filtered.dump
-diff -b -w ${srcdir}/filtered.cdl ./filtered.dump
+diff -b -w ${srcdir}/ref_filtered.cdl ./filtered.dump
 echo "	*** Pass: nccopy simple filter"
 
+echo "	*** Testing '*' filter application"
+${NCCOPY} -M0 -F "*,307,9,4" unfilteredvv.nc filteredvv.nc
+${NCDUMP} -s filteredvv.nc > ./tst_filtervv.txt
+# Remove irrelevant -s output
+sclean ./tst_filtervv.txt ./filteredvv.dump
+diff -b -w ${srcdir}/ref_filteredvv.cdl ./filteredvv.dump
+echo "	*** Pass: nccopy '*' filter"
+
+echo "	*** Testing 'v|v' filter application"
+${NCCOPY} -M0 -F "var1|/g/var2,307,9,4" unfilteredvv.nc filteredvbar.nc
+${NCDUMP} -n filteredvv -s filteredvbar.nc > ./tst_filtervbar.txt
+# Remove irrelevant -s output
+sclean ./tst_filtervbar.txt ./filteredvbar.dump
+diff -b -w ${srcdir}/ref_filteredvv.cdl ./filteredvbar.dump
+echo "	*** Pass: nccopy 'v|v' filter"
+
 echo "	*** Testing pass-thru of filters"
 rm -f ./tst_filter.txt tst_filter2.txt ./tst_filter2.nc
 # Prevent failure by allowing any chunk size
@@ -170,6 +194,8 @@ rm -f ./bzip*.nc ./unfiltered.nc ./filtered.nc ./tst_filter.txt ./tst_filter2.tx
 rm -f ./test_bzip2.c
 rm -f ./testmisc.nc
 rm -f ./tst_filter2.nc
+rm -f ./unfilteredvv.nc ./filteredvv.nc ./filteredvbar.nc
+rm -f ./tst_filtervv.txt ./tst_filtervbar.txt
 echo "*** Pass: all selected tests passed"
 
 exit 0

ncdump/list.c

@@ -36,6 +36,19 @@ List* listnew(void)
   return l;
 }
 
+int
+listfreeall(List* l)
+{
+  if(l) {
+    int i;
+    for(i=0;i<listlength(l);i++) {
+	void* elem = listget(l,i);
+	if(elem != NULL) free(elem);
+    }
+  }
+  return listfree(l);
+}
+
 int
 listfree(List* l)
 {

ncdump/list.h

@@ -21,6 +21,7 @@ typedef struct List {
 
 EXTERNC List* listnew(void);
 EXTERNC int listfree(List*);
+EXTERNC int listfreeall(List*);
 EXTERNC int listsetalloc(List*,unsigned long);
 EXTERNC int listsetlength(List*,unsigned long);
 

ncdump/nccopy.1

@@ -273,20 +273,25 @@ Set the log level; only usable if nccopy supports netCDF-4 (enhanced).
 Set the minimum chunk size; only usable if nccopy supports netCDF-4 (enhanced).
 .IP "\fB \-F \fP \fIfilterspec\fP"
 For netCDF-4 output, including netCDF-4 classic model, specify a filter
-to apply to an specified variable in the output. As a rule, the filter
+to apply to a specified set of variables in the output. As a rule, the filter
 is a compression/decompression algorithm with a unique numeric identifier
 assigned by the HDF Group (see https://support.hdfgroup.org/services/filters.html).
 .IP
 The \fIfilterspec\fP argument has this general form.
 .RS
-fqn,filterid,param1,param2...paramn
+fqn1|fqn2...,filterid,param1,param2...paramn
+or
+*,filterid,param1,param2...paramn
 .RE
-The fqn (fully qualified name) is the name
+An fqn (fully qualified name) is the name
 of a variable prefixed by its containing
 groups with the group names separated by forward slash ('/').
 An example might be \FI/g1/g2/var\fP. Alternatively,
 just the variable name can be given if it is in the root group:
 e.g. \FIvar\fP. Backslash escapes may be used as needed.
+A note of warning: the '|' separator is a bash reserved character, so you will
+probably need to put the filter spec in some kind of quotes or otherwise escape it.
+.IP
 The filterid is an unsigned positive integer representing the id
 assigned by the HDFgroup to the filter. Following the id is a sequence of
 parameters defining the operation of the filter. Each parameter