test(fdb-list): add marsflow tests

tests/fdb/tools/list/CMakeLists.txt

@@ -1,8 +1,29 @@
-set(test_name fdb_list)
+# get test data for the list tests
+ecbuild_get_test_multidata( TARGET get_fdb_test_data_list LABELS list NAMES "od.oper.grib" )
 
-configure_file( ${test_name}.sh.in ${test_name}.sh @ONLY )
+# list of tests
+list( APPEND fdb_tools_list_tests
+    all
+    all_full
+    location
+    masking
+    minimum_keys
+    porcelain
+    ranges
+    depth
+)
 
-ecbuild_add_test(
-    TYPE SCRIPT
-    CONDITION HAVE_FDB_BUILD_TOOLS
-    COMMAND ${test_name}.sh)
+# add the tests
+foreach( _test ${fdb_tools_list_tests} )
+
+    configure_file( list_${_test}.sh.in list_${_test}.sh @ONLY )
+
+    ecbuild_add_test(
+        TARGET test_fdb_tools_list_${_test}
+        TYPE SCRIPT
+        LABELS list
+        CONDITION HAVE_FDB_BUILD_TOOLS
+        TEST_DEPENDS get_fdb_test_data_list
+        COMMAND list_${_test}.sh )
+
+endforeach( )

tests/fdb/tools/list/list_all.sh.in

@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+
+set -ux
+
+yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
+die() { yell "$*"; exit 1; }
+try() { "$@" || die "Errored HERE => '$*'"; }
+
+export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
+export FDB5_CONFIG_FILE="local.yaml"
+export FDB_HOME=@PROJECT_BINARY_DIR@
+
+test_name=all
+src_data="od.oper.grib"
+src_dir=@CMAKE_CURRENT_SOURCE_DIR@
+bin_dir=@CMAKE_CURRENT_BINARY_DIR@
+
+########################################################################################################################
+
+echo "running test '$test_name' on $(hostname)"
+
+try cd $bin_dir
+
+rm -rf $test_name || true
+mkdir -p $test_name/localroot
+
+try cd $test_name
+
+try cp "$src_dir/local.yaml" ./
+try cp "$bin_dir/$src_data" ./
+
+########################################################################################################################
+
+count() {
+  [[ $# -eq 1 ]] || die "count requires 1 argument; expected count"
+  val=$(wc -l < out) && val=$((val + 0))
+  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
+}
+
+grep_count() {
+  [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
+  val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
+  [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
+}
+
+########################################################################################################################
+
+# Ensure that listing finds the correct data, but excludes duplicates.
+
+# Set up the data for the test
+
+try grib_set -s class=rd,expver=xxxx,type=fc,step=0 "$src_data" data.xxxx.0.grib
+try grib_set -s class=rd,expver=xxxx,type=fc,step=1 "$src_data" data.xxxx.1.grib
+try grib_set -s class=rd,expver=xxxx,type=fc,step=2 "$src_data" data.xxxx.2.grib
+try grib_set -s class=rd,expver=xxxy,type=fc,step=0 "$src_data" data.xxxy.0.grib
+try grib_set -s class=rd,expver=xxxy,type=fc,step=1 "$src_data" data.xxxy.1.grib
+try grib_set -s class=rd,expver=xxxy,type=fc,step=2 "$src_data" data.xxxy.2.grib
+
+# Set up some regexes for later testing
+
+regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}"
+regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}"
+regex_x2="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=2,levelist=[0-9]+,param=[0-9]+}"
+regex_y0="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}"
+regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}"
+regex_y2="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=2,levelist=[0-9]+,param=[0-9]+}"
+
+# We loop over all these tests twice. We should get the same listing results both times
+# as the default behaviour is to only show the data that would be retrieved (i.e. that matches a MARS request.)
+
+sum=0
+for i in 1 2; do
+
+  echo "============ Loop $i ============"
+
+  try fdb-write data.xxxx.0.grib
+  try fdb-list --all --minimum-keys="" --porcelain > out
+  if [[ $i = 1 ]]; then
+    sum=$((sum + 24))
+    count $sum
+    grep_count "$regex_x1" 0
+    grep_count "$regex_x2" 0
+    grep_count "$regex_y0" 0
+    grep_count "$regex_y1" 0
+    grep_count "$regex_y2" 0
+  fi
+  grep_count "$regex_x0" 24
+
+  try fdb-write data.xxxy.0.grib
+  try fdb-list --all --minimum-keys="" --porcelain > out
+  if [[ $i = 1 ]]; then
+    sum=$((sum + 24))
+    count $sum
+    grep_count "$regex_x1" 0
+    grep_count "$regex_x2" 0
+    grep_count "$regex_y1" 0
+    grep_count "$regex_y2" 0
+  fi
+  grep_count "$regex_x0" 24
+  grep_count "$regex_y0" 24
+
+
+  try fdb-write data.xxxx.1.grib
+  try fdb-list --all --minimum-keys="" --porcelain > out
+  if [[ $i = 1 ]]; then
+    sum=$((sum + 24))
+    count $sum
+    grep_count "$regex_x2" 0
+    grep_count "$regex_y1" 0
+    grep_count "$regex_y2" 0
+  fi
+  grep_count "$regex_x0" 24
+  grep_count "$regex_y0" 24
+  grep_count "$regex_x1" 24
+
+  try fdb-write data.xxxy.1.grib
+  try fdb-list --all --minimum-keys="" --porcelain > out
+  if [[ $i = 1 ]]; then
+    sum=$((sum + 24))
+    count $sum
+    grep_count "$regex_x2" 0
+    grep_count "$regex_y2" 0
+  fi
+  grep_count "$regex_x0" 24
+  grep_count "$regex_y0" 24
+  grep_count "$regex_x1" 24
+  grep_count "$regex_y1" 24
+
+
+  try fdb-write data.xxxx.2.grib
+  try fdb-list --all --minimum-keys="" --porcelain > out
+  if [[ $i = 1 ]]; then
+    sum=$((sum + 24))
+    count $sum
+    grep_count "$regex_y2" 0
+  fi
+  grep_count "$regex_x0" 24
+  grep_count "$regex_y0" 24
+  grep_count "$regex_x1" 24
+  grep_count "$regex_y1" 24
+  grep_count "$regex_x2" 24
+
+  try fdb-write data.xxxy.2.grib
+  try fdb-list --all --minimum-keys="" --porcelain > out
+  [[ $i = 1 ]] && sum=$((sum + 24))
+  count $sum
+  grep_count "$regex_x0" 24
+  grep_count "$regex_y0" 24
+  grep_count "$regex_x1" 24
+  grep_count "$regex_y1" 24
+  grep_count "$regex_x2" 24
+  grep_count "$regex_y2" 24
+
+done

tests/fdb/tools/list/list_all_full.sh.in

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+
+set -ux
+
+yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
+die() { yell "$*"; exit 1; }
+try() { "$@" || die "Errored HERE => '$*'"; }
+
+export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
+export FDB5_CONFIG_FILE="local.yaml"
+export FDB_HOME=@PROJECT_BINARY_DIR@
+
+test_name=all_full
+src_data="od.oper.grib"
+src_dir=@CMAKE_CURRENT_SOURCE_DIR@
+bin_dir=@CMAKE_CURRENT_BINARY_DIR@
+
+########################################################################################################################
+
+echo "running test '$test_name' on $(hostname)"
+
+try cd $bin_dir
+
+rm -rf $test_name || true
+mkdir -p $test_name/localroot
+
+try cd $test_name
+
+try cp "$src_dir/local.yaml" ./
+try cp "$bin_dir/$src_data" ./
+
+########################################################################################################################
+
+count() {
+  [[ $# -eq 1 ]] || die "count requires 1 argument; expected count"
+  val=$(wc -l < out) && val=$((val + 0))
+  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
+}
+
+grep_count() {
+  [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
+  val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
+  [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
+}
+
+########################################################################################################################
+
+# Ensure that listing finds the correct data, but excludes duplicates.
+# This is the same data as in the all test, but with a full enumeration
+
+# Set up the data for the test
+
+try grib_set -s class=rd,expver=xxxx,type=fc,step=0 "$src_data" data.xxxx.0.grib
+try grib_set -s class=rd,expver=xxxx,type=fc,step=1 "$src_data" data.xxxx.1.grib
+try grib_set -s class=rd,expver=xxxy,type=fc,step=0 "$src_data" data.xxxy.0.grib
+try grib_set -s class=rd,expver=xxxy,type=fc,step=1 "$src_data" data.xxxy.1.grib
+
+# Set up some regexes for later testing
+
+regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}"
+regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}"
+regex_y0="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}"
+regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}"
+
+# We loop over all these tests twice. We should get the same listing results both times
+# as the default behaviour is to only show the data that would be retrieved (i.e. that matches a MARS request.)
+
+for i in 0 1; do
+
+  try fdb-write data.xxxx.0.grib
+  try fdb-list --all --minimum-keys="" --porcelain --full > out
+  count $((24+(i*96)))
+  grep_count "$regex_x0" $((24+(24*i)))
+  grep_count "$regex_x1" $((0+(24*i)))
+  grep_count "$regex_y0" $((0+(24*i)))
+  grep_count "$regex_y1" $((0+(24*i)))
+
+  try fdb-write data.xxxy.0.grib
+  try fdb-list --all --minimum-keys="" --porcelain --full > out
+  count $((48+(i*96)))
+  grep_count "$regex_x0" $((24+(24*i)))
+  grep_count "$regex_x1" $((0+(24*i)))
+  grep_count "$regex_y0" $((24+(24*i)))
+  grep_count "$regex_y1" $((0+(24*i)))
+
+  try fdb-write data.xxxx.1.grib
+  try fdb-list --all --minimum-keys="" --porcelain --full > out
+  count $((72+(i*96)))
+  grep_count "$regex_x0" $((24+(24*i)))
+  grep_count "$regex_x1" $((24+(24*i)))
+  grep_count "$regex_y0" $((24+(24*i)))
+  grep_count "$regex_y1" $((0+(24*i)))
+
+  try fdb-write data.xxxy.1.grib
+  try fdb-list --all --minimum-keys="" --porcelain --full > out
+  count $((96+(i*96)))
+  grep_count "$regex_x0" $((24+(24*i)))
+  grep_count "$regex_x1" $((24+(24*i)))
+  grep_count "$regex_y0" $((24+(24*i)))
+  grep_count "$regex_y1" $((24+(24*i)))
+
+done
+
+exit 0

tests/fdb/tools/list/fdb_list.sh.in → tests/fdb/tools/list/list_depth.sh.in

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-# set -eu
+set -ux
 
-yell() { echo "$(basename "$0"): $*" >&2; }
+yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
 die() { yell "$*"; exit 1; }
 try() { "$@" || die "Errored HERE => '$*'"; }
 
@@ -14,42 +14,46 @@ export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
 export FDB5_CONFIG_FILE="local.yaml"
 export FDB_HOME=@PROJECT_BINARY_DIR@
 
-tname=list_test
-srcdir=@CMAKE_CURRENT_SOURCE_DIR@
-bindir=@CMAKE_CURRENT_BINARY_DIR@
+test_name=depth
+src_data=x.grib
+src_dir=@CMAKE_CURRENT_SOURCE_DIR@
+bin_dir=@CMAKE_CURRENT_BINARY_DIR@
 
 ########################################################################################################################
 
-try cd $bindir
+echo "running test '$test_name' on $(hostname)"
 
-rm -rf $tname || true
-mkdir -p $tname/localroot
+try cd $bin_dir
 
-try cd $tname
+rm -rf $test_name || true
+mkdir -p $test_name/localroot
 
-for f in local.yaml x.grib; do
-  cp "$srcdir/$f" ./
+try cd $test_name
+
+for f in local.yaml $src_data; do
+  try cp "$src_dir/$f" ./
 done
 
 ########################################################################################################################
 # Create a grib file with 4 steps and populate FDB
 
-try grib_set -s step=0 x.grib 0.grib
-try grib_set -s step=6 x.grib 6.grib
-try grib_set -s step=9 x.grib 9.grib
+try grib_set -s step=0 $src_data 0.grib
+try grib_set -s step=6 $src_data 6.grib
+try grib_set -s step=9 $src_data 9.grib
 
 for f in *.grib; do
   try grib_set -s type=an "$f" "cf.$f"
 done
 
 for f in *.grib; do
-  try cat "$f" >> "$tname.grib"
+  try cat "$f" >> "$test_name.grib"
 done
 
-try fdb-write "$tname.grib"
+try fdb-write "$test_name.grib"
 
 ########################################################################################################################
 # test all
+
 out=$(fdb-list --all --minimum-keys= --porcelain)
 exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=0,param=166}
 {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166}