From 118baccfefef67a529d31ee13cf6621a8e081635 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 28 Jan 2024 15:52:44 -0700 Subject: [PATCH] upload --- RELEASE_NOTES.md | 1 + docs/nczarr.md | 126 ++++++----- libnczarr/zinternal.h | 40 +++- nczarr_test/Makefile.am | 291 +++++++++++++++++++++++++- v3_tests/Makefile.am | 299 ++++++++++++++++++++++++++- v3_tests/ref_power_901_constants.cdl | 38 ---- v3_tests/ref_tst_nans.dmp | 16 -- 7 files changed, 682 insertions(+), 129 deletions(-) delete mode 100644 v3_tests/ref_power_901_constants.cdl delete mode 100644 v3_tests/ref_tst_nans.dmp diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index f1e25239a6..9107c02949 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Add experimental support for the Zarr Version 3 storage format. This code willl change as the Zarr Version 3 Specification evolves. * Added infrastructure to allow for `CMAKE_UNITY_BUILD`, (thanks \@jschueller). See [Github #2839](https://github.com/Unidata/netcdf-c/pull/2839) for more information. * [cmake] Move dependency management out of the root-level `CMakeLists.txt` into two different files in the `cmake/` folder, `dependencies.cmake` and `netcdf_functions_macros.cmake`. See [Github #2838](https://github.com/Unidata/netcdf-c/pull/2838/) for more information. * Fix some problems in handling S3 urls with missing regions. See [Github #2819](https://github.com/Unidata/netcdf-c/pull/2819). diff --git a/docs/nczarr.md b/docs/nczarr.md index 7fe0b5e816..f5cb2a1199 100644 --- a/docs/nczarr.md +++ b/docs/nczarr.md @@ -708,14 +708,13 @@ The content of these objects is the same as the contents of the corresponding ke # Appendix F. Zarr Version 3: NCZarr Version 3 Meta-Data Representation. {#nczarr_version3} For Zarr version 3, the added NCZarr specific metadata is stored in a different way than in Zarr version 2. -Specifically, the following Netcdf information needs to be captured by NCZarr: +Specifically, the following Netcdf-4 meta-data information needs to be captured by NCZarr: 1. Shared dimensions: name and size. 2. Unlimited dimensions: which dimensions are unlimited. 3. Attribute types. - 4. Netcdf types not included in Zarr: currently only "char". + 4. Netcdf types not included in Zarr: currently "char" and "string". 5. Zarr types not included in Netcdf: currently only "complex(32|64)" -As with NCZarr version 2, the above information is captured by adding special dictionary keys in various locations -in the standard Zarr version 3 objects. +This extra netcdfd-4 meta-data to attributes so as to not interfere with existing implementations. ## Supported Types Zarr version 3 supports the following "atomic" types: @@ -725,16 +724,15 @@ It also defines two structured type: complex64 and complex128. NCZarr supports all of the atomic types. Specialized support is provided for the following Netcdf types: char, string. -Specialized support is also provided for the following -Zarr types: bool, complex64. -The type complex128 is not supported. +The Zarr types bool and complex64 are not yet supported, but will be added shortly. +The type complex128 is not supported at all. -The Zarr type "bool" appears in the netcdf types as +The Zarr type "bool" will appear in the netcdf types as the enum type "_bool" whose netcdf declaration is as follows: ```` ubyte enum _bool_t {FALSE=0, TRUE=1}; ```` -The type complex64 is supported by by defining this compound type: +The type complex64 will be supported by by defining this compound type: ```` compound _Complex64_t { float64 i; float64 j;} ```` @@ -749,25 +747,52 @@ and its maximum size is specified. So, the Netcdf types "char" and "string" are stored in the Zarr file as of type "uint8" and "r<8*n>", respectively where _n_ is the maximum length of the string in bytes (not characters). -The fact that they represent "char" and "string" is encoded in the "_nczarr_array" key (see below). +The fact that they represent "char" and "string" is encoded in the "_nczarr_array" attribute (see below). ## NCZarr Superblock -The primary repository for NCZarr metadata is in the _zarr.info_ object in the root group of the Zarr file. -Within that object, the following Dictionary key and corresponding JSON value is stored. +The *_nczarr_superblock* attribute is used as a useful marker to signal that a file is in fact NCZarr as opposed to Zarr. +This attribute is stored in the *zarr.info* attributes in the root group of the Zarr file. +The relevant attribute has the following format: ```` "_nczarr_superblock": { - "nczarr_format": "3.0.0", - "dimensions": { - "": {"size": , "unlimited": 1|0}, "": {"size": , "unlimited": 1|0} ... - }, - "groups": ["", "", ...], - "arrays": ["", "", ...], + "version": "3.0.0", + format": 3 } ```` -The "dimensions" key holds information about all the shared dimensions across -all groups. This aggregation improves performance by not requiring all groups -to be searched looking for dimension information. -Similarly, the gross structure of the subgroups and variables (aka arrays) is captured. +## Group Annotations +The optional *_nczarr_group* attribute is stored in the attributes of a Zarr group within +the *zarr.json* object in that group. +The relevant attribute has the following format: +```` +"_nczarr_group": { +\"dimensions\": [{name: , size: , unlimited: 1|0},...], +\"arrays\": ["",...], +\"subgroups\": ["",...] +} +```` +Its purpose is two-fold: +1. record the objects immediately within that group +2. define netcdf-4 dimenension objects within that group. + +## Array Annotations +In order to support Netcdf concepts in Zarr, it may be necessary +to annotate a Zarr array with extra information. +The optional *_nczarr_array* attribute is stored in the attributes of a Zarr array within +the *zarr.json* object in that array. +The relevant attribute has the following format: +```` +"_nczarr_array": { +\"dimension_references\": [\"/g1/g2/d1\", \"/d2\",...], +\"type_alias\": "" // optional +} +```` +The *dimension_references* key is an expansion of the "dimensions" key +found in the *zarr.json* object for an array. +The problem with "dimensions" is that it specifies a simple name for each +dimension, whereas netcdf-4 requires that the array references dimension objects +that may appear in groups anywhere in the file. These references are encoded +as FQNs "pointing" to a specific dimension declaration (see *_nczarr_group* attribute +defined previously). FQN is an acronym for "Fully Qualified Name". It is a series of names separated by the "/" character, much @@ -778,22 +803,22 @@ Similarly ````/g1/g2/d2```` defines a dimension "d2" defined in the group g2, which in turn is a subgroup of group g1, which is a subgroup of the root group. -## Array Annotations -In order to support Netcdf concepts in Zarr, it may be necessary -to annotate a Zarr array with extra information. -The form this takes is to add the following key and JSON value -to the _zarr.info_ array object. -```` -"_nczarr_array": { - "nczarr_format": "3.0.0", - "nczarr_type: "char"|"string" - } -} -```` -The "nczarr_type"_ key indicates how to re-interpret -the array's type as a corresponding NCZarr/Netcdf type. - -## Attribute Typing +The *type-alias* key is used to annotate the type of an array +to allow discovery of netcdf-4 specific types. +Specifically, there are three current cases: +| dtype | type_alias | +| ----- | ---------- | +| uint8 | char | +| rn | string | +| uint8 | json | + +If, for example, an array's dtype is specified as *uint8*, then it may be that +it is actually of unsigned 8-bit integer type. But it may actually be of some +netcdf-4 type that is encoded as *uint8* in order to be recognized by other -- pure zarr-- +implementations. So, for example, if the netcdf-4 type is *char*, then the array's +dtype is *uint8*, but its type alias is *char*. + +## Attribute Type Annotation In Zarr version 2, attributes are stored in a separate _.zattr_ object. In Zarr version 3, group and array attributes are now stored inside the corresponding _zarr.info_. object under the dictionary key "attributes". @@ -801,24 +826,27 @@ Note that this decision is still under discussion and it may be changed to store attributes in an object separate from _zarr.info_. Regardless of where the attributes are stored, and in order to -support Netcdf typed attributes, the per-attribute information +support netcdf-4 typed attributes, the per-attribute information is stored as a special attribute called _\_nczarr_attrs\__ defined to hold NCZarr specific attribute information. Currently, it only holds the attribute typing information. +It can appear in any *zarr.json* object: group or array. -Its JSON form is this: +Its form is this: ```` "_nczarr_attrs": { - "nczarr_format": "3.0.0", - {"types": { - "": , - "": , - ... - } + "attribute_types": [ + {"name": "attr1", "configuration": {"type": ""}}, + ... + ] } ```` -There is one entry for every regular attribute giving the type +There is one entry for every attribute (including itself) giving the type of that attribute. +It should be noted that Zarr allows the value of an attribute to be an arbitrary +JSON-encoded structure. In order to support this in netcdf-4, is such a structure +is encountered as an attribute value, then it typed as *json* (see previously +described table). ## Codec Specification The Zarr version 3 representation of codecs is slightly different @@ -840,7 +868,7 @@ intended to be a detailed chronology. Rather, it provides highlights that will be of interest to NCZarr users. In order to see exact changes, It is necessary to use the 'git diff' command. -## 11/02/2023 +## 01/31/2024 1. Add description of support for Zarr version 3 as an appendix. ## 3/10/2023 @@ -862,4 +890,4 @@ include arbitrary JSON expressions; see Appendix D for more details. __Author__: Dennis Heimbigner
__Email__: dmh at ucar dot edu
__Initial Version__: 4/10/2020
-__Last Revised__: 11/01/2023 +__Last Revised__: 01/31/2024 diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 03db8479b3..98c8744d60 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -97,12 +97,9 @@ Inserted into any .zattrs ? or should it go into the container? Inserted into root group zarr.json as an extra attribute. _nczarr_superblock: { "version": 3.0.0, - "format": 3, + "format": 3 } -Optionally Inserted into any group zarr.json as an extra attribute. -"_nczarr_attrs": {\"attribute_types\": [{\"name\": \"attr1\", \"configuration\": {\"type\": \"\"}}, ...]} - Optionally inserted into any group zarr.json as an attribute: "_nczarr_group": "{ \"dimensions\": [{name: , size: , unlimited: 1|0},...], @@ -111,12 +108,31 @@ Optionally inserted into any group zarr.json as an attribute: }" Optionally Inserted into any array zarr.json as an attribute: +```` "_nczarr_array": "{ -\"dimensions_references\": [\"/g1/g2/d1\", \"/d2\",...], +\"dimension_references\": [\"/g1/g2/d1\", \"/d2\",...], \"type_alias\": "" // optional }" - -The "type-alias key is used to signal ambiguous dtypes. +```` +The *dimension_references* key is an expansion of the "dimensions" key +found in the *zarr.json* object for an array. +The problem with "dimensions" is that it specifies a simple name for each +dimension, whereas netcdf-4 requires that the array references dimension objects +that may appear in groups anywhere in the file. These references are encoded +as FQNs "pointing" to a specific dimension declaration (see *_nczarr_group* attribute +defined previously). + +FQN is an acronym for "Fully Qualified Name". +It is a series of names separated by the "/" character, much +like a file system path. +It identifies the group in which the dimension is ostensibly "defined" in the Netcdf sense. +For example ````/d1```` defines a dimension "d1" defined in the root group. +Similarly ````/g1/g2/d2```` defines a dimension "d2" defined in the +group g2, which in turn is a subgroup of group g1, which is a subgroup +of the root group. + +The *type-alias* key is used to annotate the type of an array +to allow discovery of netcdf-4 specific types. Specifically, there are three current cases: | dtype | type_alias | | ----- | ---------- | @@ -124,8 +140,16 @@ Specifically, there are three current cases: | rn | string | | uint8 | json | -Optionally Inserted into any array zarr.json as an extra attribute. +If, for example, an array's dtype is specified as *uint8*, then it may be that +it is actually of unsigned 8-bit integer type. But it may actually be of some +netcdf-4 type that is encoded as *uint8* in order to be recognized by other -- pure zarr-- +implementations. So, for example, if the netcdf-4 type is *char*, then the array's +dtype is *uint8*, but its type alias is *char*. + + +Optionally Inserted into any group zarr.json or array zarr.json is the extra attribute. "_nczarr_attrs": {\"attribute_types\": [{\"name\": \"attr1\", \"configuration\": {\"type\": \"\"}}, ...]} + */ #define NCZ_V2_SUPERBLOCK "_nczarr_superblock" diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index 6fe070bbc6..88ced66089 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -4,6 +4,21 @@ # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am +TESTS_ENVIRONMENT = +TEST_EXTENSIONS = .sh +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT += export SETX=1; + +AM_CFLAGS += -I${top_srcdir} +AM_CPPFLAGS += -I${top_srcdir} +AM_CPPFLAGS += -I${top_srcdir}/libnczarr +AM_CPPFLAGS += -I${top_srcdir}/plugins +LDADD = ${top_builddir}/liblib/libnetcdf.la +AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la +AM_CXXFLAGS = + BUILT_SOURCES = EXTRA_DIST = DISTCLEANFILES = @@ -12,14 +27,6 @@ check_PROGRAMS = include $(top_srcdir)/nczarr_test/Makefile.files -BUILT_SOURCES += ${DISTBUILTSCRIPTS} - -# V2 specific files -EXTRA_DIST += CMakeLists.txt Makefile.core Makefile.files ${DISTV2BASELINE} ${DISTV2SCRIPTS} \ - ${DISTBASELINE} ${DISTSCRIPTS} - -include $(top_srcdir)/nczarr_test/Makefile.core - if BUILD_UTILITIES # Test v2 specific tests @@ -40,5 +47,273 @@ endif #BUILD_UTILITIES DISTCLEANFILES += findplugin.sh ${BUILT_SOURCES} +testcommonsrc = test_utils.c test_utils.h +commonsrc = ut_util.c ut_test.c ut_includes.h ut_test.h ut_util.h test_nczarr_utils.h + +check_PROGRAMS += ut_map ut_mapapi ut_json +ut_map_SOURCES = ut_map.c ${commonsrc} +ut_mapapi_SOURCES = ut_mapapi.c ${commonsrc} +ut_json_SOURCES = ut_json.c ${commonsrc} + +test_fillonlyz_SOURCES = test_fillonlyz.c ${testcommonsrc} + +check_PROGRAMS += test_fillonlyz test_quantize test_notzarr + +# Unlimited Dimension tests +if USE_HDF5 +test_put_vars_two_unlim_dim_SOURCES = test_put_vars_two_unlim_dim.c ${testcommonsrc} +check_PROGRAMS += test_zchunks test_zchunks2 test_zchunks3 test_unlim_vars test_put_vars_two_unlim_dim +check_PROGRAMS += test_unlim_io +test_unlim_io_SOURCES = test_unlim_io.c ${testcommonsrc} +TESTS += test_put_vars_two_unlim_dim +endif + +if BUILD_UTILITIES + +TESTS += run_ut_misc.sh +TESTS += run_ut_map.sh +TESTS += run_ut_mapapi.sh + +if AX_DISABLE # Obsolete tests +TESTS += run_ut_chunk.sh +endif + +TESTS += run_ncgen4.sh run_nan.sh + +TESTS += run_quantize.sh +TESTS += run_purezarr.sh +TESTS += run_misc.sh +TESTS += run_jsonconvention.sh +TESTS += run_strings.sh +TESTS += run_scalar.sh +TESTS += run_nulls.sh +TESTS += run_notzarr.sh + +# Unlimited Dimension tests (at least in part) +TESTS += run_mud.sh +TESTS += test_unlim_vars + +if USE_HDF5 +# Unlimited Dimension tests +check_PROGRAMS += test_chunking +TESTS += run_nccopy5.sh +endif + +if USE_HDF5 +TESTS += run_unlim_io.sh run_nccopyz.sh +endif + +if LARGE_FILE_TESTS + check_PROGRAMS += test_writecaching test_readcaching + TESTS += run_cachetest.sh +endif + +endif #BUILD_UTILITIES + +if BUILD_UTILITIES + +if ENABLE_NCZARR_ZIP + TESTS += run_newformat.sh +endif + +if LARGE_FILE_TESTS +check_PROGRAMS += test_chunkcases +test_chunkcases_SOURCES = test_chunkcases.c ${testcommonsrc} +TESTS += run_chunkcases.sh +endif + +check_PROGRAMS += test_endians +TESTS += test_endians + +if USE_HDF5 +TESTS += run_fillonlyz.sh +endif + +if BUILD_BENCHMARKS + +UTILSRC = bm_utils.c timer_utils.c test_utils.c bm_utils.h bm_timer.h + +bm_chunks3_SOURCES = bm_chunks3.c ${UTILSRC} + +check_PROGRAMS += bm_chunks3 + +# The perf tests need modernization +if AX_DISABLE +TESTS += run_perf_chunks1.sh +endif + +endif # BUILD_BENCHMARKS + +if ENABLE_NCZARR_FILTERS +if ENABLE_FILTER_TESTING + +check_PROGRAMS += test_filter_avail test_filter_vlen +TESTS += run_filter_vlen.sh run_filter_misc.sh + +# Do simple filter functionality tests for nczarr +check_PROGRAMS += test_nczfilter +TESTS += run_nczfilter.sh + +# Echo filter tests from nc_test4 +check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi +TESTS += run_filter.sh +TESTS += run_specific_filters.sh + +if AX_DISABLE +# This test is too dangerous to run in a parallel make environment. +# It causes race conditions. So suppress and only test by hand. +TESTS += run_unknown.sh +endif + +if ISMINGW +XFAIL_TESTS = run_filter.sh run_specific_filters.sh run_nczfilter.sh +endif # ISMINGW + +endif #ENABLE_FILTER_TESTING +endif #ENABLE_NCZARR_FILTERS + +endif #BUILD_UTILITIES + +# These programs are used by the test cases +noinst_PROGRAMS = zhex +zhex_SOURCES = zhex.c +noinst_PROGRAMS += zisjson +zisjson_SOURCES = zisjson.c +noinst_PROGRAMS += zmapio +zmapio_SOURCES = zmapio.c +noinst_PROGRAMS += zs3parse +zs3parse_SOURCES = zs3parse.c + +if ENABLE_S3 +noinst_PROGRAMS += s3util +s3util_SOURCES = s3util.c +if ENABLE_S3_TESTALL +if ENABLE_S3_AWS +check_PROGRAMS += tst_pure_awssdk +tst_pure_awssdk_SOURCES = tst_pure_awssdk.cpp +AM_CXXFLAGS += -std=c++11 +endif +endif +endif + +if ENABLE_NCDUMPCHUNKS +# Given a netcdf4|NCZarr file, dump the actual chunk contents. +# Used to validate nczarr chunking code. +AM_CPPFLAGS += -I$(top_srcdir)/libnczarr +noinst_PROGRAMS += ncdumpchunks +ncdumpchunks_SOURCES = ncdumpchunks.c +endif + +CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp*.zmap tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip + +BUILT_SOURCES += run_unknown.sh run_specific_filters.sh run_filter_vlen.sh run_filterinstall.sh run_mud.sh run_nccopy5.sh run_filter_misc.sh test_quantize.c test_filter_vlen.c test_unlim_vars.c test_endians.c test_put_vars_two_unlim_dim.c test_chunking.c test_put_vars_two_unlim_dim.c + +test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_quantize.c >> $@ + +test_filter_vlen.c: $(top_srcdir)/nc_test4/tst_filter_vlen.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_filter_vlen.c >> $@ + +test_unlim_vars.c: $(top_srcdir)/nc_test4/tst_unlim_vars.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_unlim_vars.c >> $@ + +test_endians.c: $(top_srcdir)/nc_test4/tst_h5_endians.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_h5_endians.c >> $@ + +test_put_vars_two_unlim_dim.c: $(top_srcdir)/nc_test4/tst_put_vars_two_unlim_dim.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_put_vars_two_unlim_dim.c >> $@ + +test_chunking.c: $(top_srcdir)/ncdump/tst_chunking.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/ncdump/tst_chunking.c >> $@ + +run_unknown.sh: $(top_srcdir)/nc_test4/tst_unknown.sh + rm -f $@ run_unknown.tmp + echo "#!/bin/bash" > run_unknown.tmp + echo "TESTNCZARR=1" >> run_unknown.tmp + cat $(top_srcdir)/nc_test4/tst_unknown.sh >> run_unknown.tmp + tr -d '\r' < run_unknown.tmp > $@ + chmod a+x $@ + rm -f run_unknown.tmp + +run_specific_filters.sh: $(top_srcdir)/nc_test4/tst_specific_filters.sh + rm -f $@ run_specific_filters.tmp + echo "#!/bin/bash" > run_specific_filters.tmp + echo "TESTNCZARR=1" >> run_specific_filters.tmp + cat $(top_srcdir)/nc_test4/tst_specific_filters.sh >> run_specific_filters.tmp + tr -d '\r' < run_specific_filters.tmp > $@ + chmod a+x $@ + rm -f run_specific_filters.tmp + +run_filter_vlen.sh: $(top_srcdir)/nc_test4/tst_filter_vlen.sh + rm -f $@ run_filter_vlen.tmp + echo "#!/bin/bash" > run_filter_vlen.tmp + echo "TESTNCZARR=1" >> run_filter_vlen.tmp + cat $(top_srcdir)/nc_test4/tst_filter_vlen.sh >> run_filter_vlen.tmp + tr -d '\r' < run_filter_vlen.tmp > $@ + chmod a+x $@ + rm -f run_filter_vlen.tmp + +run_filter_misc.sh: $(top_srcdir)/nc_test4/tst_filter_misc.sh + rm -f $@ run_filter_misc.tmp + echo "#!/bin/bash" > run_filter_misc.tmp + echo "TESTNCZARR=1" >> run_filter_misc.tmp + cat $(top_srcdir)/nc_test4/tst_filter_misc.sh >> run_filter_misc.tmp + tr -d '\r' < run_filter_misc.tmp > $@ + chmod a+x $@ + rm -f run_filter_misc.tmp + +run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh + rm -f $@ run_filterinstall.tmp + echo "#!/bin/bash" > run_filterinstall.tmp + echo "TESTNCZARR=1" >> run_filterinstall.tmp + cat $(top_srcdir)/nc_test4/tst_filterinstall.sh >> run_filterinstall.tmp + tr -d '\r' < run_filterinstall.tmp > $@ + chmod a+x $@ + rm -f run_filterinstall.tmp + +run_mud.sh: $(top_srcdir)/ncdump/tst_mud.sh + rm -f $@ run_mud.tmp + echo "#!/bin/bash" > run_mud.tmp + echo "TESTNCZARR=1" >> run_mud.tmp + cat $(top_srcdir)/ncdump/tst_mud.sh >> run_mud.tmp + tr -d '\r' < run_mud.tmp > $@ + chmod a+x $@ + rm -f run_mud.tmp + +run_nccopy5.sh: $(top_srcdir)/ncdump/tst_nccopy5.sh + rm -f $@ run_nccopy5.tmp + echo "#!/bin/bash" > run_nccopy5.tmp + echo "TESTNCZARR=1" >> run_nccopy5.tmp + cat $(top_srcdir)/ncdump/tst_nccopy5.sh >> run_nccopy5.tmp + tr -d '\r' < run_nccopy5.tmp > $@ + chmod a+x $@ + rm -f run_nccopy5.tmp + +EXTRA_DIST += CMakeLists.txt run_corrupt.sh run_external.sh run_interop.sh run_nczarr_fill.sh run_chunkcases.sh run_fillonlyz.sh run_filter.sh run_nccopyz.sh run_nczarr_fill.sh run_nczfilter.sh run_newformat.sh run_perf_chunks1.sh run_unlim_io.sh test_nczarr.sh run_ut_map.sh run_ut_mapapi.sh run_ncgen4.sh run_nan.sh run_quantize.sh run_purezarr.sh run_misc.sh run_jsonconvention.sh run_strings.sh run_scalar.sh run_nulls.sh run_notzarr.sh + +EXTRA_DIST += ref_noshape.file.zip ref_power_901_constants.cdl ref_power_901_constants_orig.zip ref_avail1.cdl ref_avail1.dmp ref_avail1.txt ref_byte.cdl ref_byte_fill_value_null.cdl ref_byte_fill_value_null.zarr.zip ref_byte.zarr.zip ref_fillonly.cdl ref_groups.h5 ref_misc1.cdl ref_misc1.dmp ref_ndims.cdl ref_ndims.dmp ref_newformatpure.cdl ref_notzarr.tar.gz ref_nulls.cdl ref_nulls_nczarr.baseline ref_oldformat.cdl ref_oldformat.zip ref_perdimspecs.cdl ref_purezarr_base.cdl ref_quotes.cdl ref_quotes_orig.zip ref_rem.cdl ref_rem.dmp ref_scalar.cdl ref_skip.cdl ref_skip.txt ref_skipw.cdl ref_string.cdl ref_t_meta_dim1.cdl ref_t_meta_var1.cdl ref_ut_json_build.txt ref_ut_json_parse.txt ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_mapapi_search.txt ref_ut_map_create.cdl ref_ut_map_readmeta2.txt ref_ut_map_readmeta.txt ref_ut_map_search.txt ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl ref_ut_proj.txt ref_ut_testmap_create.cdl ref_whole.cdl ref_whole.txt ref_zarr_test_data.cdl.gz + +# Remove directories +clean-local: + rm -fr testdir_* testset_* + rm -fr tmp_*.nc tmp_*.zarr tst_quantize*.zarr tmp*.file results.file results.s3 results.zip + rm -fr rcmiscdir ref_power_901_constants.file + rm -fr v3_*.log v3_*.trs +if ENABLE_S3_TESTALL + bash ${abs_top_builddir}/s3cleanup.sh +endif + # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ diff --git a/v3_tests/Makefile.am b/v3_tests/Makefile.am index 2fb2b2f122..b58042c135 100644 --- a/v3_tests/Makefile.am +++ b/v3_tests/Makefile.am @@ -4,30 +4,309 @@ # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am +AM_TESTS_ENVIRONMENT += export NCZARRFORMAT=3; +TESTS_ENVIRONMENT = +TEST_EXTENSIONS = .sh +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT += export SETX=1; + +AM_CFLAGS += -I${top_srcdir} +AM_CPPFLAGS += -I${top_srcdir} +AM_CPPFLAGS += -I${top_srcdir}/libnczarr +AM_CPPFLAGS += -I${top_srcdir}/plugins +LDADD = ${top_builddir}/liblib/libnetcdf.la +AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la +AM_CXXFLAGS = + BUILT_SOURCES = EXTRA_DIST = DISTCLEANFILES = TESTS = check_PROGRAMS = +NCZARRFILES = -# Following files are V3 specific versions -V3FILES = ${DISTV3BASELINE} ${DISTV3SCRIPTS} +testcommonsrc = test_utils.c test_utils.h +commonsrc = ut_util.c ut_test.c ut_includes.h ut_test.h ut_util.h test_nczarr_utils.h -include $(top_srcdir)/nczarr_test/Makefile.files -include $(top_srcdir)/nczarr_test/Makefile.core +check_PROGRAMS += ut_map ut_mapapi ut_json +ut_map_SOURCES = ut_map.c ${commonsrc} +ut_mapapi_SOURCES = ut_mapapi.c ${commonsrc} +ut_json_SOURCES = ut_json.c ${commonsrc} -AM_TESTS_ENVIRONMENT += export NCZARRFORMAT=3; +test_fillonlyz_SOURCES = test_fillonlyz.c ${testcommonsrc} + +check_PROGRAMS += test_fillonlyz test_quantize test_notzarr + +# Unlimited Dimension tests +if USE_HDF5 +test_put_vars_two_unlim_dim_SOURCES = test_put_vars_two_unlim_dim.c ${testcommonsrc} +check_PROGRAMS += test_zchunks test_zchunks2 test_zchunks3 test_unlim_vars test_put_vars_two_unlim_dim +check_PROGRAMS += test_unlim_io +test_unlim_io_SOURCES = test_unlim_io.c ${testcommonsrc} +TESTS += test_put_vars_two_unlim_dim +endif + +if BUILD_UTILITIES + +TESTS += run_ut_misc.sh +TESTS += run_ut_map.sh +TESTS += run_ut_mapapi.sh + +if AX_DISABLE # Obsolete tests +TESTS += run_ut_chunk.sh +endif + +TESTS += run_ncgen4.sh run_nan.sh + +TESTS += run_quantize.sh +TESTS += run_purezarr.sh +TESTS += run_misc.sh +TESTS += run_jsonconvention.sh +TESTS += run_strings.sh +TESTS += run_scalar.sh +TESTS += run_nulls.sh +TESTS += run_notzarr.sh + +# Unlimited Dimension tests (at least in part) +TESTS += run_mud.sh +TESTS += test_unlim_vars + +if USE_HDF5 +# Unlimited Dimension tests +check_PROGRAMS += test_chunking +TESTS += run_nccopy5.sh +endif + +if USE_HDF5 +TESTS += run_unlim_io.sh run_nccopyz.sh +endif + +if LARGE_FILE_TESTS + check_PROGRAMS += test_writecaching test_readcaching + TESTS += run_cachetest.sh +endif + +endif #BUILD_UTILITIES + +if BUILD_UTILITIES + +if ENABLE_NCZARR_ZIP + TESTS += run_newformat.sh +endif + +if LARGE_FILE_TESTS +check_PROGRAMS += test_chunkcases +test_chunkcases_SOURCES = test_chunkcases.c ${testcommonsrc} +TESTS += run_chunkcases.sh +endif + +check_PROGRAMS += test_endians +TESTS += test_endians + +if USE_HDF5 +TESTS += run_fillonlyz.sh +endif + +if BUILD_BENCHMARKS + +UTILSRC = bm_utils.c timer_utils.c test_utils.c bm_utils.h bm_timer.h + +bm_chunks3_SOURCES = bm_chunks3.c ${UTILSRC} + +check_PROGRAMS += bm_chunks3 + +# The perf tests need modernization +if AX_DISABLE +TESTS += run_perf_chunks1.sh +endif + +endif # BUILD_BENCHMARKS + +if ENABLE_NCZARR_FILTERS +if ENABLE_FILTER_TESTING -EXTRA_DIST += CMakeLists.txt ${V3FILES} +check_PROGRAMS += test_filter_avail test_filter_vlen +TESTS += run_filter_vlen.sh run_filter_misc.sh -FILES = ${DISTSRC} ${DISTSCRIPTS} ${DISTBASELINE} findplugin.sh +# Do simple filter functionality tests for nczarr +check_PROGRAMS += test_nczfilter +TESTS += run_nczfilter.sh -DISTCLEANFILES += ${FILES} ${DISTBUILTSRC} ${DISTBUILTSCRIPTS} +# Echo filter tests from nc_test4 +check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi +TESTS += run_filter.sh +TESTS += run_specific_filters.sh -BUILT_SOURCES += ${FILES} +if AX_DISABLE +# This test is too dangerous to run in a parallel make environment. +# It causes race conditions. So suppress and only test by hand. +TESTS += run_unknown.sh +endif -${FILES}: +if ISMINGW +XFAIL_TESTS = run_filter.sh run_specific_filters.sh run_nczfilter.sh +endif # ISMINGW + +endif #ENABLE_FILTER_TESTING +endif #ENABLE_NCZARR_FILTERS + +endif #BUILD_UTILITIES + +# These programs are used by the test cases +noinst_PROGRAMS = zhex +zhex_SOURCES = zhex.c +noinst_PROGRAMS += zisjson +zisjson_SOURCES = zisjson.c +noinst_PROGRAMS += zmapio +zmapio_SOURCES = zmapio.c +noinst_PROGRAMS += zs3parse +zs3parse_SOURCES = zs3parse.c + +if ENABLE_S3 +noinst_PROGRAMS += s3util +s3util_SOURCES = s3util.c +if ENABLE_S3_TESTALL +if ENABLE_S3_AWS +check_PROGRAMS += tst_pure_awssdk +tst_pure_awssdk_SOURCES = tst_pure_awssdk.cpp +AM_CXXFLAGS += -std=c++11 +endif +endif +endif + +if ENABLE_NCDUMPCHUNKS +# Given a netcdf4|NCZarr file, dump the actual chunk contents. +# Used to validate nczarr chunking code. +AM_CPPFLAGS += -I$(top_srcdir)/libnczarr +noinst_PROGRAMS += ncdumpchunks +ncdumpchunks_SOURCES = ncdumpchunks.c +endif + +CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp*.zmap tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip + +BUILT_SOURCES += run_unknown.sh run_specific_filters.sh run_filter_vlen.sh run_filterinstall.sh run_mud.sh run_nccopy5.sh run_filter_misc.sh test_quantize.c test_filter_vlen.c test_unlim_vars.c test_endians.c test_put_vars_two_unlim_dim.c test_chunking.c test_put_vars_two_unlim_dim.c + +test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_quantize.c >> $@ + +test_filter_vlen.c: $(top_srcdir)/nc_test4/tst_filter_vlen.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_filter_vlen.c >> $@ + +test_unlim_vars.c: $(top_srcdir)/nc_test4/tst_unlim_vars.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_unlim_vars.c >> $@ + +test_endians.c: $(top_srcdir)/nc_test4/tst_h5_endians.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_h5_endians.c >> $@ + +test_put_vars_two_unlim_dim.c: $(top_srcdir)/nc_test4/tst_put_vars_two_unlim_dim.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/nc_test4/tst_put_vars_two_unlim_dim.c >> $@ + +test_chunking.c: $(top_srcdir)/ncdump/tst_chunking.c + rm -f $@ + echo "#define TESTNCZARR" > $@ + cat $(top_srcdir)/ncdump/tst_chunking.c >> $@ + +run_unknown.sh: $(top_srcdir)/nc_test4/tst_unknown.sh + rm -f $@ run_unknown.tmp + echo "#!/bin/bash" > run_unknown.tmp + echo "TESTNCZARR=1" >> run_unknown.tmp + cat $(top_srcdir)/nc_test4/tst_unknown.sh >> run_unknown.tmp + tr -d '\r' < run_unknown.tmp > $@ + chmod a+x $@ + rm -f run_unknown.tmp + +run_specific_filters.sh: $(top_srcdir)/nc_test4/tst_specific_filters.sh + rm -f $@ run_specific_filters.tmp + echo "#!/bin/bash" > run_specific_filters.tmp + echo "TESTNCZARR=1" >> run_specific_filters.tmp + cat $(top_srcdir)/nc_test4/tst_specific_filters.sh >> run_specific_filters.tmp + tr -d '\r' < run_specific_filters.tmp > $@ + chmod a+x $@ + rm -f run_specific_filters.tmp + +run_filter_vlen.sh: $(top_srcdir)/nc_test4/tst_filter_vlen.sh + rm -f $@ run_filter_vlen.tmp + echo "#!/bin/bash" > run_filter_vlen.tmp + echo "TESTNCZARR=1" >> run_filter_vlen.tmp + cat $(top_srcdir)/nc_test4/tst_filter_vlen.sh >> run_filter_vlen.tmp + tr -d '\r' < run_filter_vlen.tmp > $@ + chmod a+x $@ + rm -f run_filter_vlen.tmp + +run_filter_misc.sh: $(top_srcdir)/nc_test4/tst_filter_misc.sh + rm -f $@ run_filter_misc.tmp + echo "#!/bin/bash" > run_filter_misc.tmp + echo "TESTNCZARR=1" >> run_filter_misc.tmp + cat $(top_srcdir)/nc_test4/tst_filter_misc.sh >> run_filter_misc.tmp + tr -d '\r' < run_filter_misc.tmp > $@ + chmod a+x $@ + rm -f run_filter_misc.tmp + +run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh + rm -f $@ run_filterinstall.tmp + echo "#!/bin/bash" > run_filterinstall.tmp + echo "TESTNCZARR=1" >> run_filterinstall.tmp + cat $(top_srcdir)/nc_test4/tst_filterinstall.sh >> run_filterinstall.tmp + tr -d '\r' < run_filterinstall.tmp > $@ + chmod a+x $@ + rm -f run_filterinstall.tmp + +run_mud.sh: $(top_srcdir)/ncdump/tst_mud.sh + rm -f $@ run_mud.tmp + echo "#!/bin/bash" > run_mud.tmp + echo "TESTNCZARR=1" >> run_mud.tmp + cat $(top_srcdir)/ncdump/tst_mud.sh >> run_mud.tmp + tr -d '\r' < run_mud.tmp > $@ + chmod a+x $@ + rm -f run_mud.tmp + +run_nccopy5.sh: $(top_srcdir)/ncdump/tst_nccopy5.sh + rm -f $@ run_nccopy5.tmp + echo "#!/bin/bash" > run_nccopy5.tmp + echo "TESTNCZARR=1" >> run_nccopy5.tmp + cat $(top_srcdir)/ncdump/tst_nccopy5.sh >> run_nccopy5.tmp + tr -d '\r' < run_nccopy5.tmp > $@ + chmod a+x $@ + rm -f run_nccopy5.tmp + +# Built Sources that are copies over from nczarr_test +NCZARRFILES += findplugin.sh run_corrupt.sh run_external.sh run_interop.sh run_chunkcases.sh run_fillonlyz.sh run_filter.sh run_nccopyz.sh run_nczarr_fill.sh run_nczfilter.sh run_newformat.sh run_perf_chunks1.sh run_unlim_io.sh test_nczarr.sh run_ut_map.sh run_ut_mapapi.sh run_ncgen4.sh run_quantize.sh run_purezarr.sh run_misc.sh run_jsonconvention.sh run_strings.sh run_scalar.sh run_nulls.sh run_notzarr.sh + +NCZARRFILES += ref_noshape.file.zip ref_power_901_constants.cdl ref_power_901_constants_orig.zip ref_avail1.cdl ref_avail1.dmp ref_avail1.txt ref_byte.cdl ref_byte_fill_value_null.cdl ref_byte_fill_value_null.zarr.zip ref_byte.zarr.zip ref_fillonly.cdl ref_groups.h5 ref_misc1.cdl ref_misc1.dmp ref_ndims.cdl ref_ndims.dmp ref_newformatpure.cdl ref_notzarr.tar.gz ref_nulls.cdl ref_nulls_nczarr.baseline ref_oldformat.cdl ref_oldformat.zip ref_perdimspecs.cdl ref_purezarr_base.cdl ref_quotes.cdl ref_quotes_orig.zip ref_rem.cdl ref_rem.dmp ref_scalar.cdl ref_skip.cdl ref_skip.txt ref_skipw.cdl ref_string.cdl ref_t_meta_dim1.cdl ref_t_meta_var1.cdl ref_ut_json_build.txt ref_ut_json_parse.txt ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_mapapi_search.txt ref_ut_map_create.cdl ref_ut_map_readmeta2.txt ref_ut_map_readmeta.txt ref_ut_map_search.txt ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl ref_ut_proj.txt ref_ut_testmap_create.cdl ref_whole.cdl ref_whole.txt ref_zarr_test_data.cdl.gz ref_tst_nans.dmp + +${NCZARRFILES}: for u in $@ ; do cp -f ${top_srcdir}/nczarr_test/$$u $(top_srcdir)/v3_tests ; done +BUILT_SOURCES += ${NCZARRFILES} + +# Files that are not built +EXTRA_DIST += CMakeLists.txt +# V3 specific files +EXTRA_DIST += ref_nczarr2zarr.cdl ref_purezarr.cdl ref_xarray.cdl ref_misc2.cdl ref_jsonconvention.cdl ref_jsonconvention.zmap ref_nulls_zarr.baseline ref_string_zarr.baseline ref_string_nczarr.baseline ref_zarr_test_data_2d.cdl.gz ref_groups_regular.cdl ref_bzip2.cdl ref_filtered.cdl ref_any.cdl ref_multi.cdl run_nan.sh + +DISTCLEANFILES += findplugin.sh ${BUILT_SOURCES} + +# Remove directories +clean-local: + rm -fr testdir_* testset_* + rm -fr tmp_*.nc tmp_*.zarr tst_quantize*.zarr tmp*.file results.file results.s3 results.zip + rm -fr rcmiscdir ref_power_901_constants.file + rm -fr v3_*.log v3_*.trs +if ENABLE_S3_TESTALL + bash ${abs_top_builddir}/s3cleanup.sh +endif + # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ diff --git a/v3_tests/ref_power_901_constants.cdl b/v3_tests/ref_power_901_constants.cdl deleted file mode 100644 index d69e283b4b..0000000000 --- a/v3_tests/ref_power_901_constants.cdl +++ /dev/null @@ -1,38 +0,0 @@ -netcdf ref_power_901_constants { -dimensions: - lat = 361 ; - lon = 576 ; -variables: - float FRLAKE(lat, lon) ; - FRLAKE:fmissing_value = 999999986991104. ; - FRLAKE:long_name = "fraction_of_lake" ; - FRLAKE:standard_name = "fraction_of_lake" ; - FRLAKE:units = "1" ; - FRLAKE:valid_range = -999999986991104., 999999986991104. ; - FRLAKE:vmax = 999999986991104. ; - FRLAKE:vmin = -999999986991104. ; - float FRLAND(lat, lon) ; - FRLAND:fmissing_value = 999999986991104. ; - FRLAND:long_name = "fraction_of_land" ; - FRLAND:standard_name = "fraction_of_land" ; - FRLAND:units = "1" ; - FRLAND:valid_range = -999999986991104., 999999986991104. ; - FRLAND:vmax = 999999986991104. ; - FRLAND:vmin = -999999986991104. ; - float FROCEAN(lat, lon) ; - FROCEAN:fmissing_value = 999999986991104. ; - FROCEAN:long_name = "fraction_of_ocean" ; - FROCEAN:standard_name = "fraction_of_ocean" ; - FROCEAN:units = "1" ; - FROCEAN:valid_range = -999999986991104., 999999986991104. ; - FROCEAN:vmax = 999999986991104. ; - FROCEAN:vmin = -999999986991104. ; - float FRLANDICE(lat, lon) ; - FRLANDICE:fmissing_value = 999999986991104. ; - FRLANDICE:long_name = "fraction_of_land_ice" ; - FRLANDICE:standard_name = "fraction_of_land_ice" ; - FRLANDICE:units = "1" ; - FRLANDICE:valid_range = -999999986991104., 999999986991104. ; - FRLANDICE:vmax = 999999986991104. ; - FRLANDICE:vmin = -999999986991104. ; -} diff --git a/v3_tests/ref_tst_nans.dmp b/v3_tests/ref_tst_nans.dmp deleted file mode 100644 index 18b5d40a91..0000000000 --- a/v3_tests/ref_tst_nans.dmp +++ /dev/null @@ -1,16 +0,0 @@ -netcdf ref_tst_nans { -dimensions: - dim = 3 ; -variables: - float fvar(dim) ; - fvar:att = -Infinityf, NaNf, Infinityf ; - fvar:_FillValue = NaNf ; - double dvar(dim) ; - dvar:att = -Infinity, NaN, Infinity ; - dvar:_FillValue = NaN ; -data: - - fvar = -Infinityf, _, Infinityf ; - - dvar = -Infinity, _, Infinity ; -}