From 13eeff94c0c11a29de969e0734341db9b1663855 Mon Sep 17 00:00:00 2001
From: Metin Cakircali <metin.cakircali@ecmwf.int>
Date: Fri, 13 Sep 2024 11:48:54 +0200
Subject: [PATCH] test(fdb-list): fix macos grep issue

and several minor issues
---
 tests/fdb/tools/list/list_all.sh.in          |  58 +++++------
 tests/fdb/tools/list/list_all_full.sh.in     |  52 +++++-----
 tests/fdb/tools/list/list_depth.sh.in        |  34 +++----
 tests/fdb/tools/list/list_location.sh.in     |  50 ++++-----
 tests/fdb/tools/list/list_masking.sh.in      |  67 ++++++------
 tests/fdb/tools/list/list_minimum_keys.sh.in |  68 +++++--------
 tests/fdb/tools/list/list_porcelain.sh.in    | 101 ++++++++++---------
 tests/fdb/tools/list/list_ranges.sh.in       |  76 +++++++-------
 8 files changed, 240 insertions(+), 266 deletions(-)

diff --git a/tests/fdb/tools/list/list_all.sh.in b/tests/fdb/tools/list/list_all.sh.in
index 8a2f1b2e1..d714c4b59 100755
--- a/tests/fdb/tools/list/list_all.sh.in
+++ b/tests/fdb/tools/list/list_all.sh.in
@@ -2,10 +2,24 @@
 
 set -ux
 
-yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
+yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
 die() { yell "$*"; exit 1; }
 try() { "$@" || die "Errored HERE => '$*'"; }
 
+line_count() {
+  [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
+  val=$(wc -l < out) && val=$((val + 0))
+  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
+}
+
+grep_count() {
+  [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
+  val=$(grep -cE "$1" out)
+  [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
+}
+
+########################################################################################################################
+
 export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
 export FDB5_CONFIG_FILE="local.yaml"
 export FDB_HOME=@PROJECT_BINARY_DIR@
@@ -21,8 +35,8 @@ echo "running test '$test_name' on $(hostname)"
 
 try cd $bin_dir
 
-rm -rf $test_name || true
-mkdir -p $test_name/localroot
+try rm -rf $test_name
+try mkdir -p $test_name/localroot
 
 try cd $test_name
 
@@ -31,20 +45,6 @@ try cp "$bin_dir/$src_data" ./
 
 ########################################################################################################################
 
-count() {
-  [[ $# -eq 1 ]] || die "count requires 1 argument; expected count"
-  val=$(wc -l < out) && val=$((val + 0))
-  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
-}
-
-grep_count() {
-  [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
-  val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
-  [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
-}
-
-########################################################################################################################
-
 # Ensure that listing finds the correct data, but excludes duplicates.
 
 # Set up the data for the test
@@ -74,10 +74,10 @@ for i in 1 2; do
   echo "============ Loop $i ============"
 
   try fdb-write data.xxxx.0.grib
-  try fdb-list --all --minimum-keys="" --porcelain > out
+  try fdb-list --all --minimum-keys="" --porcelain | tee out
   if [[ $i = 1 ]]; then
     sum=$((sum + 24))
-    count $sum
+    line_count $sum
     grep_count "$regex_x1" 0
     grep_count "$regex_x2" 0
     grep_count "$regex_y0" 0
@@ -87,10 +87,10 @@ for i in 1 2; do
   grep_count "$regex_x0" 24
 
   try fdb-write data.xxxy.0.grib
-  try fdb-list --all --minimum-keys="" --porcelain > out
+  try fdb-list --all --minimum-keys="" --porcelain | tee out
   if [[ $i = 1 ]]; then
     sum=$((sum + 24))
-    count $sum
+    line_count $sum
     grep_count "$regex_x1" 0
     grep_count "$regex_x2" 0
     grep_count "$regex_y1" 0
@@ -101,10 +101,10 @@ for i in 1 2; do
 
 
   try fdb-write data.xxxx.1.grib
-  try fdb-list --all --minimum-keys="" --porcelain > out
+  try fdb-list --all --minimum-keys="" --porcelain | tee out
   if [[ $i = 1 ]]; then
     sum=$((sum + 24))
-    count $sum
+    line_count $sum
     grep_count "$regex_x2" 0
     grep_count "$regex_y1" 0
     grep_count "$regex_y2" 0
@@ -114,10 +114,10 @@ for i in 1 2; do
   grep_count "$regex_x1" 24
 
   try fdb-write data.xxxy.1.grib
-  try fdb-list --all --minimum-keys="" --porcelain > out
+  try fdb-list --all --minimum-keys="" --porcelain | tee out
   if [[ $i = 1 ]]; then
     sum=$((sum + 24))
-    count $sum
+    line_count $sum
     grep_count "$regex_x2" 0
     grep_count "$regex_y2" 0
   fi
@@ -128,10 +128,10 @@ for i in 1 2; do
 
 
   try fdb-write data.xxxx.2.grib
-  try fdb-list --all --minimum-keys="" --porcelain > out
+  try fdb-list --all --minimum-keys="" --porcelain | tee out
   if [[ $i = 1 ]]; then
     sum=$((sum + 24))
-    count $sum
+    line_count $sum
     grep_count "$regex_y2" 0
   fi
   grep_count "$regex_x0" 24
@@ -141,9 +141,9 @@ for i in 1 2; do
   grep_count "$regex_x2" 24
 
   try fdb-write data.xxxy.2.grib
-  try fdb-list --all --minimum-keys="" --porcelain > out
+  try fdb-list --all --minimum-keys="" --porcelain | tee out
   [[ $i = 1 ]] && sum=$((sum + 24))
-  count $sum
+  line_count $sum
   grep_count "$regex_x0" 24
   grep_count "$regex_y0" 24
   grep_count "$regex_x1" 24
diff --git a/tests/fdb/tools/list/list_all_full.sh.in b/tests/fdb/tools/list/list_all_full.sh.in
index 51300eb46..79963ebda 100755
--- a/tests/fdb/tools/list/list_all_full.sh.in
+++ b/tests/fdb/tools/list/list_all_full.sh.in
@@ -2,10 +2,24 @@
 
 set -ux
 
-yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
+yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
 die() { yell "$*"; exit 1; }
 try() { "$@" || die "Errored HERE => '$*'"; }
 
+line_count() {
+  [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
+  val=$(wc -l < out) && val=$((val + 0))
+  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
+}
+
+grep_count() {
+  [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
+  val=$(grep -cEE "$1" out)
+  [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
+}
+
+########################################################################################################################
+
 export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
 export FDB5_CONFIG_FILE="local.yaml"
 export FDB_HOME=@PROJECT_BINARY_DIR@
@@ -21,8 +35,8 @@ echo "running test '$test_name' on $(hostname)"
 
 try cd $bin_dir
 
-rm -rf $test_name || true
-mkdir -p $test_name/localroot
+try rm -rf $test_name
+try mkdir -p $test_name/localroot
 
 try cd $test_name
 
@@ -31,20 +45,6 @@ try cp "$bin_dir/$src_data" ./
 
 ########################################################################################################################
 
-count() {
-  [[ $# -eq 1 ]] || die "count requires 1 argument; expected count"
-  val=$(wc -l < out) && val=$((val + 0))
-  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
-}
-
-grep_count() {
-  [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
-  val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
-  [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
-}
-
-########################################################################################################################
-
 # Ensure that listing finds the correct data, but excludes duplicates.
 # This is the same data as in the all test, but with a full enumeration
 
@@ -67,33 +67,35 @@ regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}
 
 for i in 0 1; do
 
+  echo "============ Loop $i ============"
+
   try fdb-write data.xxxx.0.grib
-  try fdb-list --all --minimum-keys="" --porcelain --full > out
-  count $((24+(i*96)))
+  try fdb-list --all --minimum-keys="" --porcelain --full | tee out
+  line_count $((24+(i*96)))
   grep_count "$regex_x0" $((24+(24*i)))
   grep_count "$regex_x1" $((0+(24*i)))
   grep_count "$regex_y0" $((0+(24*i)))
   grep_count "$regex_y1" $((0+(24*i)))
 
   try fdb-write data.xxxy.0.grib
-  try fdb-list --all --minimum-keys="" --porcelain --full > out
-  count $((48+(i*96)))
+  try fdb-list --all --minimum-keys="" --porcelain --full | tee out
+  line_count $((48+(i*96)))
   grep_count "$regex_x0" $((24+(24*i)))
   grep_count "$regex_x1" $((0+(24*i)))
   grep_count "$regex_y0" $((24+(24*i)))
   grep_count "$regex_y1" $((0+(24*i)))
 
   try fdb-write data.xxxx.1.grib
-  try fdb-list --all --minimum-keys="" --porcelain --full > out
-  count $((72+(i*96)))
+  try fdb-list --all --minimum-keys="" --porcelain --full | tee out
+  line_count $((72+(i*96)))
   grep_count "$regex_x0" $((24+(24*i)))
   grep_count "$regex_x1" $((24+(24*i)))
   grep_count "$regex_y0" $((24+(24*i)))
   grep_count "$regex_y1" $((0+(24*i)))
 
   try fdb-write data.xxxy.1.grib
-  try fdb-list --all --minimum-keys="" --porcelain --full > out
-  count $((96+(i*96)))
+  try fdb-list --all --minimum-keys="" --porcelain --full | tee out
+  line_count $((96+(i*96)))
   grep_count "$regex_x0" $((24+(24*i)))
   grep_count "$regex_x1" $((24+(24*i)))
   grep_count "$regex_y0" $((24+(24*i)))
diff --git a/tests/fdb/tools/list/list_depth.sh.in b/tests/fdb/tools/list/list_depth.sh.in
index fbb6c12c2..9a40e52be 100755
--- a/tests/fdb/tools/list/list_depth.sh.in
+++ b/tests/fdb/tools/list/list_depth.sh.in
@@ -2,13 +2,11 @@
 
 set -ux
 
-yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
+yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
 die() { yell "$*"; exit 1; }
 try() { "$@" || die "Errored HERE => '$*'"; }
 
-function request() {
-  echo "RETRIEVE,CLASS=OD,TYPE=CF,STREAM=ENFO,EXPVER=0001,LEVTYPE=PL,DATE=20201106,TIME=1200,STEP=00,DOMAIN=G,$1" > req;
-}
+########################################################################################################################
 
 export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
 export FDB5_CONFIG_FILE="local.yaml"
@@ -25,8 +23,8 @@ echo "running test '$test_name' on $(hostname)"
 
 try cd $bin_dir
 
-rm -rf $test_name || true
-mkdir -p $test_name/localroot
+try rm -rf $test_name
+try mkdir -p $test_name/localroot
 
 try cd $test_name
 
@@ -52,9 +50,8 @@ done
 try fdb-write "$test_name.grib"
 
 ########################################################################################################################
-# test all
 
-out=$(fdb-list --all --minimum-keys= --porcelain)
+echo "Test: all"
 exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=0,param=166}
 {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166}
 {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=6,param=166}
@@ -63,33 +60,34 @@ exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc
 {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166}
 {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166}
 {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}"
+out=$(try fdb-list --all --minimum-keys= --porcelain)
 try test "$exp" = "$out"
 
-# test date depth=1
-out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=1)
+echo "Test: date depth=1"
 exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}"
+out=$(try fdb-list date=20201102 --minimum-keys="" --porcelain --depth=1)
 try test "$exp" = "$out"
 
-# test date depth=2
-out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2)
+echo "Test: date depth=2"
 exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}
 {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}"
+out=$(try fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2)
 try test "$exp" = "$out"
 
-# test cf depth=1
-out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=1)
+echo "Test: cf depth=1"
 exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}"
+out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=1)
 try test "$exp" = "$out"
 
-# test cf depth=2
-out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=2)
+echo "Test: cf depth=2"
 exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}"
+out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=2)
 try test "$exp" = "$out"
 
-# test cf depth=3
-out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=3)
+echo "Test: cf depth=3"
 exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166}
 {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166}
 {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166}
 {class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}"
+out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=3)
 try test "$exp" = "$out"
diff --git a/tests/fdb/tools/list/list_location.sh.in b/tests/fdb/tools/list/list_location.sh.in
index 5a8903888..063a6a29a 100755
--- a/tests/fdb/tools/list/list_location.sh.in
+++ b/tests/fdb/tools/list/list_location.sh.in
@@ -2,10 +2,24 @@
 
 set -ux
 
-yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
+yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
 die() { yell "$*"; exit 1; }
 try() { "$@" || die "Errored HERE => '$*'"; }
 
+line_count() {
+  [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
+  val=$(wc -l < out) && val=$((val + 0))
+  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
+}
+
+grep_count() {
+  [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
+  val=$(grep -cE "$1" out)
+  [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
+}
+
+########################################################################################################################
+
 export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
 export FDB5_CONFIG_FILE="local.yaml"
 export FDB_HOME=@PROJECT_BINARY_DIR@
@@ -21,8 +35,8 @@ echo "running test '$test_name' on $(hostname)"
 
 try cd $bin_dir
 
-rm -rf $test_name || true
-mkdir -p $test_name/localroot
+try rm -rf $test_name
+try mkdir -p $test_name/localroot
 
 try cd $test_name
 
@@ -31,37 +45,23 @@ try cp "$bin_dir/$src_data" ./
 
 ########################################################################################################################
 
-count() {
-  [[ $# -eq 1 ]] || die "count requires 1 argument; expected count"
-  val=$(wc -l < out) && val=$((val + 0))
-  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
-}
-
-grep_count() {
-  [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
-  val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
-  [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
-}
-
-########################################################################################################################
-
 # Check that we can obtain the location of the data
 
 try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib
 try grib_ls -m data.xxxx.grib
 try fdb-write data.xxxx.grib
 
-try fdb-list --all --minimum-keys= --porcelain --full > out
-count 24
+echo "Test: --all --minimum-keys= --porcelain --full"
+try fdb-list --all --minimum-keys= --porcelain --full | tee out
+line_count 24
 
-# Specify the location flag
 
-try fdb-list class=rd,expver=xxxx,time=0000 --location > out
+echo "Test: Specify the location flag"
+try fdb-list class=rd,expver=xxxx,time=0000 --location | tee out
 grep_count "localroot/rd:xxxx" 12
 
-# And without the location flag, none of the location info is included
 
-fdb-list class=rd,expver=xxxx,time=0000 --porcelain
-try fdb-list class=rd,expver=xxxx,time=0000 --porcelain > out
-count 12
+echo "Test: without the location flag, none of the location info is included"
+try fdb-list class=rd,expver=xxxx,time=0000 --porcelain | tee out
+line_count 12
 grep_count "{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=0000,domain=g}{type=an,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" 12
diff --git a/tests/fdb/tools/list/list_masking.sh.in b/tests/fdb/tools/list/list_masking.sh.in
index d73c681b3..76beb44a6 100755
--- a/tests/fdb/tools/list/list_masking.sh.in
+++ b/tests/fdb/tools/list/list_masking.sh.in
@@ -2,10 +2,18 @@
 
 set -ux
 
-yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
+yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
 die() { yell "$*"; exit 1; }
 try() { "$@" || die "Errored HERE => '$*'"; }
 
+line_count() {
+  [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
+  val=$(wc -l < out) && val=$((val + 0))
+  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
+}
+
+########################################################################################################################
+
 export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
 export FDB5_CONFIG_FILE="local.yaml"
 export FDB_HOME=@PROJECT_BINARY_DIR@
@@ -21,8 +29,8 @@ echo "running test '$test_name' on $(hostname)"
 
 try cd $bin_dir
 
-rm -rf $test_name || true
-mkdir -p $test_name/localroot
+try rm -rf $test_name
+try mkdir -p $test_name/localroot
 
 try cd $test_name
 
@@ -31,52 +39,39 @@ try cp "$bin_dir/$src_data" ./
 
 ########################################################################################################################
 
-count() {
-  [[ $# -eq 1 ]] || die "count requires 1 argument; expected count"
-  val=$(wc -l < out) && val=$((val + 0))
-  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
-}
-
-# grep_count() {
-#   [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
-#   val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
-#   [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
-# }
-
-########################################################################################################################
-
 # Ensure that we can wipe specified databases with ranges in the requests
 # --> Ensures the MARS requests are working correctly
 
 try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.d1.grib
-try grib_set -s class=rd,expver=xxxx,date=20170101 "$src_data" data.xxxx.d2.grib
+try grib_set -s class=rd,expver=xxxx,date=20240912 "$src_data" data.xxxx.d2.grib
 try grib_ls -m data.xxxx.d1.grib
 
 try fdb-write data.xxxx.d1.grib
 try fdb-write data.xxxx.d2.grib
+# write the data twice
 try fdb-write data.xxxx.d1.grib
 try fdb-write data.xxxx.d2.grib
 
-echo "Incorrect number of entries written"
-try fdb-list --all --minimum-keys="" --porcelain > out
-count 48
+echo "Test: Incorrect number of entries written"
+try fdb-list --all --minimum-keys="" --porcelain | tee out
+line_count 48
 
-echo "Incorrect number of entries written"
-fdb-list --all --minimum-keys="" --porcelain --full > out
-count 96
+echo "Test: Incorrect number of entries written"
+try fdb-list --all --minimum-keys="" --porcelain --full | tee out
+line_count 96
 
-echo "Entries should be masked without --full"
-fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain > out
-count 48
+echo "Test: Entries should be masked without --full"
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain | tee out
+line_count 48
 
-echo "All entries should be visible with --full"
-try fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain --full > out
-count 96
+echo "Test: All entries should be visible with --full"
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain --full | tee out
+line_count 96
 
-echo "Entries should be masked without full"
-try fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain > out
-count 8
+echo "Test: Entries should be masked without full"
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain | tee out
+line_count 8
 
-echo "All entries should be visible with --full"
-try fdb-list class=rd,expver=xxxx,date=-1/20170101,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain --full > out
-count 16
+echo "Test: All entries should be visible with --full"
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain --full | tee out
+line_count 16
diff --git a/tests/fdb/tools/list/list_minimum_keys.sh.in b/tests/fdb/tools/list/list_minimum_keys.sh.in
index 325ca5a5c..9c62683ca 100755
--- a/tests/fdb/tools/list/list_minimum_keys.sh.in
+++ b/tests/fdb/tools/list/list_minimum_keys.sh.in
@@ -2,10 +2,18 @@
 
 set -ux
 
-yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
+yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
 die() { yell "$*"; exit 1; }
 try() { "$@" || die "Errored HERE => '$*'"; }
 
+line_count() {
+  [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
+  val=$(wc -l < out) && val=$((val + 0))
+  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
+}
+
+########################################################################################################################
+
 export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
 export FDB5_CONFIG_FILE="local.yaml"
 export FDB_HOME=@PROJECT_BINARY_DIR@
@@ -21,8 +29,8 @@ echo "running test '$test_name' on $(hostname)"
 
 try cd $bin_dir
 
-rm -rf $test_name || true
-mkdir -p $test_name/localroot
+try rm -rf $test_name
+try mkdir -p $test_name/localroot
 
 try cd $test_name
 
@@ -31,63 +39,39 @@ try cp "$bin_dir/$src_data" ./
 
 ########################################################################################################################
 
-count() {
-  [[ $# -eq 1 ]] || die "count requires 1 argument; expected count"
-  val=$(wc -l < out) && val=$((val + 0))
-  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
-}
-
-# grep_count() {
-#   [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
-#   val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
-#   [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
-# }
-
-########################################################################################################################
-
-echo "Ensure that we can wipe specified databases, but only when fully specified"
+# Ensure that we can wipe specified databases, but only when fully specified
 
 try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib
 try grib_ls -m data.xxxx.grib
-
 try fdb-write data.xxxx.grib
-try fdb-list --all --full --minimum-keys= --porcelain > out
-count 24
 
 
-echo "Check that if any of the keys are missing, then nothing happens"
+echo "Test: check all output"
+try fdb-list --all --full --minimum-keys= --porcelain | tee out
+line_count 24
 
+
+echo "Test: nothing happens if any of the keys are missing"
 for invalid_key in class=rd expver=xxxx; do
-  # set +e
   fdb-list $invalid_key
   rc=$?
-  # set -e
-
   [[ $rc = 0 ]] && die "Should not have succeeded: $invalid_key"
 done
 
 
-echo "Check that supplying the key makes it work"
+echo "Test: supplying the key"
+try fdb-list class=rd,expver=xxxx --porcelain | tee out
+line_count 24
 
-try fdb-list class=rd,expver=xxxx --porcelain > out
-count 24
 
-
-echo "Check that we can change the minimum keys"
-
-for invalid_key in class=rd,expver=xxxx \
-                   class=rd,time=1200 \
-                   expver=xxxx,time=1200; do
-  set +e
+echo "Test: changing the minimum keys"
+for invalid_key in class=rd,expver=xxxx class=rd,time=1200 expver=xxxx,time=1200; do
   fdb-list --minimum-keys="class,expver,time" $invalid_key
   rc=$?
-  set -e
-
-  [[ $rc = 0 ]] && echo "Should not have succeeded: $invalid_key" && exit 1
+  [[ $rc = 0 ]] && die "Should not have succeeded: $invalid_key"
 done
 
 
-echo "And that the purge works now this is supplied"
-
-fdb-list --minimum-keys="class,expver,time" class=rd,expver=xxxx,time=1200 --porcelain > out
-count 12
+echo "Test: the purge works now this is supplied"
+try fdb-list --minimum-keys="class,expver,time" class=rd,expver=xxxx,time=1200 --porcelain | tee out
+line_count 12
diff --git a/tests/fdb/tools/list/list_porcelain.sh.in b/tests/fdb/tools/list/list_porcelain.sh.in
index 932eb396c..e06f152ec 100755
--- a/tests/fdb/tools/list/list_porcelain.sh.in
+++ b/tests/fdb/tools/list/list_porcelain.sh.in
@@ -2,10 +2,24 @@
 
 set -ux
 
-yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
+yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
 die() { yell "$*"; exit 1; }
 try() { "$@" || die "Errored HERE => '$*'"; }
 
+line_count() {
+  [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
+  val=$(wc -l < out) && val=$((val + 0))
+  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
+}
+
+grep_count() {
+  [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
+  val=$(grep -cE "$1" out)
+  [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
+}
+
+########################################################################################################################
+
 export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
 export FDB5_CONFIG_FILE="local.yaml"
 export FDB_HOME=@PROJECT_BINARY_DIR@
@@ -21,8 +35,8 @@ echo "running test '$test_name' on $(hostname)"
 
 try cd $bin_dir
 
-rm -rf $test_name || true
-mkdir -p $test_name/localroot
+try rm -rf $test_name
+try mkdir -p $test_name/localroot
 
 try cd $test_name
 
@@ -31,20 +45,6 @@ try cp "$bin_dir/$src_data" ./
 
 ########################################################################################################################
 
-count() {
-  [[ $# -eq 1 ]] || die "count requires 1 argument; expected count"
-  val=$(wc -l < out) && val=$((val + 0))
-  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
-}
-
-grep_count() {
-  [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
-  val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
-  [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
-}
-
-########################################################################################################################
-
 # If we don't specify --porcelain, then the expanded request is prepended to the output request.
 # Measure the difference!
 
@@ -52,63 +52,64 @@ try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib
 try grib_ls -m data.xxxx.grib
 try fdb-write data.xxxx.grib
 
+tab=$'\t' # macos doesn't support '\t' in grep
 
 echo "Test all output"
 
-try fdb-list --all --minimum-keys="" --porcelain --full > out
-count 24
+try fdb-list --all --minimum-keys="" --porcelain --full | tee out
+line_count 24
 
 
 echo "Test normal output"
 
-try fdb-list class=rd,expver=xxxx > out
-grep_count '{class=rd,expver=xxxx' 24
-grep_count '^retrieve,$' 1
-grep_count '^\t.*=.*' 2
-grep_count '^\tclass=rd' 1
-grep_count '^\texpver=xxxx' 1
+try fdb-list class=rd,expver=xxxx | tee out
+grep_count "{class=rd,expver=xxxx" 24
+grep_count "^retrieve,$" 1
+grep_count "^${tab}.*=.*" 2
+grep_count "^${tab}class=rd" 1
+grep_count "^${tab}expver=xxxx" 1
 
 
 echo "Test porcelain output"
 
-try fdb-list class=rd,expver=xxxx --porcelain > out
-grep_count '{class=rd,expver=xxxx' 24
-count 24
+try fdb-list class=rd,expver=xxxx --porcelain | tee out
+line_count 24
+grep_count "{class=rd,expver=xxxx" 24
 
 
 echo "Test expansion of date and param without porcelain"
 
-try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=t > out
-grep_count '{class=rd,expver=xxxx' 0
-grep_count '^retrieve,$' 1
-grep_count '^\t.*=.*' 7
-grep_count '^\tclass=rd' 1
-grep_count '^\texpver=xxxx' 1
-grep_count '^\tparam=130' 1
-grep_count '^\tdate=[0-9]{8}' 1
+try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=t | tee out
+grep_count "{class=rd,expver=xxxx" 0
+grep_count "^retrieve,$" 1
+grep_count "^${tab}.*=.*" 7
+grep_count "^${tab}class=rd" 1
+grep_count "^${tab}expver=xxxx" 1
+grep_count "^${tab}param=130" 1
+grep_count "^${tab}date=20240911" 1
 
 
 echo "Test expansion of date and param with porcelain"
 
-try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=t --porcelain > out
-[[ "$(grep '{class=rd,expver=xxxx' out | grep 'date=[0-9]{8}' | wc -l)" -eq 0 ]] || die "Incorrect number of entries reported"
-count 0
+try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=t --porcelain | tee out
+line_count 0
+[[ "$(grep '{class=rd,expver=xxxx' out | grep 'date=20240911' | wc -l)" -eq 0 ]] || die "Incorrect number of entries reported"
 
 
 echo "Test expansion of date and param without porcelain"
 
-try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=138 > out
-[[ $(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | grep param=138 | wc -l) =~ "12" ]] || die "Incorrect number of entries reported"
-grep_count '^retrieve,$' 1
-grep_count '^\t.*=.*' 7
-grep_count '^\tclass=rd' 1
-grep_count '^\texpver=xxxx' 1
-grep_count '^\tparam=138' 1
-grep_count '^\tdate=[0-9]{8}' 1
+try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=138 | tee out
+grep_count "^retrieve,$" 1
+grep_count "^${tab}.*=.*" 7
+grep_count "^${tab}class=rd" 1
+grep_count "^${tab}expver=xxxx" 1
+grep_count "^${tab}param=138" 1
+grep_count "^${tab}date=20240911" 1
+[[ $(grep '{class=rd,expver=xxxx' out | grep 'date=20240911' | grep param=138 | wc -l) -eq 12 ]] || die "Incorrect number of entries reported"
 
 
 echo "Test expansion of date and param with porcelain"
 
-try fdb-list class=rd,expver=xxxx,date=-1,stream=oper,type=an,levtype=pl,param=138 --porcelain > out
-[[ $(grep '{class=rd,expver=xxxx' out | grep -E 'date=[0-9]{8}' | grep param=138 | wc -l) =~ "12" ]] || die "Incorrect number of entries reported"
-count 12
+try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=138 --porcelain | tee out
+line_count 12
+[[ $(grep '{class=rd,expver=xxxx' out | grep 'date=20240911' | grep param=138 | wc -l) -eq 12 ]] || die "Incorrect number of entries reported"
diff --git a/tests/fdb/tools/list/list_ranges.sh.in b/tests/fdb/tools/list/list_ranges.sh.in
index eeea01bb5..bb1812131 100755
--- a/tests/fdb/tools/list/list_ranges.sh.in
+++ b/tests/fdb/tools/list/list_ranges.sh.in
@@ -2,10 +2,18 @@
 
 set -ux
 
-yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
+yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
 die() { yell "$*"; exit 1; }
 try() { "$@" || die "Errored HERE => '$*'"; }
 
+line_count() {
+  [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
+  val=$(wc -l < out) && val=$((val + 0))
+  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
+}
+
+########################################################################################################################
+
 export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
 export FDB5_CONFIG_FILE="local.yaml"
 export FDB_HOME=@PROJECT_BINARY_DIR@
@@ -21,8 +29,8 @@ echo "running test '$test_name' on $(hostname)"
 
 try cd $bin_dir
 
-rm -rf $test_name || true
-mkdir -p $test_name/localroot
+try rm -rf $test_name
+try mkdir -p $test_name/localroot
 
 try cd $test_name
 
@@ -31,61 +39,47 @@ try cp "$bin_dir/$src_data" ./
 
 ########################################################################################################################
 
-count() {
-  [[ $# -ne 1 ]] && die "count requires 1 argument; expected count"
-  val=$(wc -l < out) && val=$((val + 0))
-  [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
-}
-
-# grep_count() {
-#   [[ $# -ne 2 ]] && die "grep_count requires 2 arguments; regex and expected count"
-#   val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
-#   [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
-# }
-
-########################################################################################################################
-
 # Ensure that we can wipe specified databases with ranges in the requests
 # --> Ensures the MARS requests are working correctly
 
 try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.d1.grib
-try grib_set -s class=rd,expver=xxxx,date=20170101 "$src_data" data.xxxx.d2.grib
-try grib_set -s class=rd,expver=xxxx,date=20180103 "$src_data" data.xxxx.d3.grib
+try grib_set -s class=rd,expver=xxxx,date=20240912 "$src_data" data.xxxx.d2.grib
+try grib_set -s class=rd,expver=xxxx,date=20241001 "$src_data" data.xxxx.d3.grib
 try grib_ls -m data.xxxx.d1.grib
 
 try fdb-write data.xxxx.d1.grib
 try fdb-write data.xxxx.d2.grib
 try fdb-write data.xxxx.d3.grib
 
-try fdb-list --all --minimum-keys="" --porcelain --full > out
-count 72
+try fdb-list --all --minimum-keys="" --porcelain --full | tee out
+line_count 72
 
-try fdb-list class=rd,expver=xxxx --porcelain > out
-count 72
+try fdb-list class=rd,expver=xxxx --porcelain | tee out
+line_count 72
 
-try fdb-list class=rd,expver=xxxx,date=-1 --porcelain > out
-count 24
+try fdb-list class=rd,expver=xxxx,date=20240911 --porcelain | tee out
+line_count 24
 
-try fdb-list class=rd,expver=xxxx,date=-1/20170101 --porcelain > out
-count 48
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912 --porcelain | tee out
+line_count 48
 
-try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103 --porcelain > out
-count 72
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001 --porcelain | tee out
+line_count 72
 
-try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103/20060101,stream=oper,type=an,levtype=pl,param=60 --porcelain > out
-count 0
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001/20060101,stream=oper,type=an,levtype=pl,param=60 --porcelain | tee out
+line_count 0
 
-try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=155 --porcelain > out
-count 36
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=155 --porcelain | tee out
+line_count 36
 
-try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155 --porcelain > out
-count 36
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155 --porcelain | tee out
+line_count 36
 
-try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138 --porcelain > out
-count 72
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155/138 --porcelain | tee out
+line_count 72
 
-try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/400/500/700/850/1000 --porcelain > out
-count 72
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/400/500/700/850/1000 --porcelain | tee out
+line_count 72
 
-try fdb-list class=rd,expver=xxxx,date=-1/20170101/20180103,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/123/1000 --porcelain > out
-count 24
+try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/123/1000 --porcelain | tee out
+line_count 24