Skip to content

Commit

Permalink
test(fdb-list): fix macos grep issue
Browse files Browse the repository at this point in the history 
and several minor issues
  • Loading branch information
@mcakircali
mcakircali committed Sep 13, 2024
1 parent d82131f commit 13eeff9
Show file tree
Hide file tree
Showing 8 changed files with 240 additions and 266 deletions.
58 changes: 29 additions & 29 deletions tests/fdb/tools/list/list_all.sh.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,24 @@

set -ux

yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
die() { yell "$*"; exit 1; }
try() { "$@" || die "Errored HERE => '$*'"; }

line_count() {
[[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
val=$(wc -l < out) && val=$((val + 0))
[[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
}

grep_count() {
[[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
val=$(grep -cE "$1" out)
[[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
}

########################################################################################################################

export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
export FDB5_CONFIG_FILE="local.yaml"
export FDB_HOME=@PROJECT_BINARY_DIR@
Expand All @@ -21,8 +35,8 @@ echo "running test '$test_name' on $(hostname)"

try cd $bin_dir

rm -rf $test_name || true
mkdir -p $test_name/localroot
try rm -rf $test_name
try mkdir -p $test_name/localroot

try cd $test_name

Expand All @@ -31,20 +45,6 @@ try cp "$bin_dir/$src_data" ./

########################################################################################################################

count() {
[[ $# -eq 1 ]] || die "count requires 1 argument; expected count"
val=$(wc -l < out) && val=$((val + 0))
[[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
}

grep_count() {
[[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
[[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
}

########################################################################################################################

# Ensure that listing finds the correct data, but excludes duplicates.

# Set up the data for the test
Expand Down Expand Up @@ -74,10 +74,10 @@ for i in 1 2; do
echo "============ Loop $i ============"

try fdb-write data.xxxx.0.grib
try fdb-list --all --minimum-keys="" --porcelain > out
try fdb-list --all --minimum-keys="" --porcelain | tee out
if [[ $i = 1 ]]; then
sum=$((sum + 24))
count $sum
line_count $sum
grep_count "$regex_x1" 0
grep_count "$regex_x2" 0
grep_count "$regex_y0" 0
Expand All @@ -87,10 +87,10 @@ for i in 1 2; do
grep_count "$regex_x0" 24

try fdb-write data.xxxy.0.grib
try fdb-list --all --minimum-keys="" --porcelain > out
try fdb-list --all --minimum-keys="" --porcelain | tee out
if [[ $i = 1 ]]; then
sum=$((sum + 24))
count $sum
line_count $sum
grep_count "$regex_x1" 0
grep_count "$regex_x2" 0
grep_count "$regex_y1" 0
Expand All @@ -101,10 +101,10 @@ for i in 1 2; do


try fdb-write data.xxxx.1.grib
try fdb-list --all --minimum-keys="" --porcelain > out
try fdb-list --all --minimum-keys="" --porcelain | tee out
if [[ $i = 1 ]]; then
sum=$((sum + 24))
count $sum
line_count $sum
grep_count "$regex_x2" 0
grep_count "$regex_y1" 0
grep_count "$regex_y2" 0
Expand All @@ -114,10 +114,10 @@ for i in 1 2; do
grep_count "$regex_x1" 24

try fdb-write data.xxxy.1.grib
try fdb-list --all --minimum-keys="" --porcelain > out
try fdb-list --all --minimum-keys="" --porcelain | tee out
if [[ $i = 1 ]]; then
sum=$((sum + 24))
count $sum
line_count $sum
grep_count "$regex_x2" 0
grep_count "$regex_y2" 0
fi
Expand All @@ -128,10 +128,10 @@ for i in 1 2; do


try fdb-write data.xxxx.2.grib
try fdb-list --all --minimum-keys="" --porcelain > out
try fdb-list --all --minimum-keys="" --porcelain | tee out
if [[ $i = 1 ]]; then
sum=$((sum + 24))
count $sum
line_count $sum
grep_count "$regex_y2" 0
fi
grep_count "$regex_x0" 24
Expand All @@ -141,9 +141,9 @@ for i in 1 2; do
grep_count "$regex_x2" 24

try fdb-write data.xxxy.2.grib
try fdb-list --all --minimum-keys="" --porcelain > out
try fdb-list --all --minimum-keys="" --porcelain | tee out
[[ $i = 1 ]] && sum=$((sum + 24))
count $sum
line_count $sum
grep_count "$regex_x0" 24
grep_count "$regex_y0" 24
grep_count "$regex_x1" 24
Expand Down
52 changes: 27 additions & 25 deletions tests/fdb/tools/list/list_all_full.sh.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,24 @@

set -ux

yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
die() { yell "$*"; exit 1; }
try() { "$@" || die "Errored HERE => '$*'"; }

line_count() {
[[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count"
val=$(wc -l < out) && val=$((val + 0))
[[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
}

grep_count() {
[[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
val=$(grep -cEE "$1" out)
[[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
}

########################################################################################################################

export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
export FDB5_CONFIG_FILE="local.yaml"
export FDB_HOME=@PROJECT_BINARY_DIR@
Expand All @@ -21,8 +35,8 @@ echo "running test '$test_name' on $(hostname)"

try cd $bin_dir

rm -rf $test_name || true
mkdir -p $test_name/localroot
try rm -rf $test_name
try mkdir -p $test_name/localroot

try cd $test_name

Expand All @@ -31,20 +45,6 @@ try cp "$bin_dir/$src_data" ./

########################################################################################################################

count() {
[[ $# -eq 1 ]] || die "count requires 1 argument; expected count"
val=$(wc -l < out) && val=$((val + 0))
[[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]"
}

grep_count() {
[[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count"
val=$(grep -E "$1" out | wc -l) && val=$((val + 0))
[[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]"
}

########################################################################################################################

# Ensure that listing finds the correct data, but excludes duplicates.
# This is the same data as in the all test, but with a full enumeration

Expand All @@ -67,33 +67,35 @@ regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}

for i in 0 1; do

echo "============ Loop $i ============"

try fdb-write data.xxxx.0.grib
try fdb-list --all --minimum-keys="" --porcelain --full > out
count $((24+(i*96)))
try fdb-list --all --minimum-keys="" --porcelain --full | tee out
line_count $((24+(i*96)))
grep_count "$regex_x0" $((24+(24*i)))
grep_count "$regex_x1" $((0+(24*i)))
grep_count "$regex_y0" $((0+(24*i)))
grep_count "$regex_y1" $((0+(24*i)))

try fdb-write data.xxxy.0.grib
try fdb-list --all --minimum-keys="" --porcelain --full > out
count $((48+(i*96)))
try fdb-list --all --minimum-keys="" --porcelain --full | tee out
line_count $((48+(i*96)))
grep_count "$regex_x0" $((24+(24*i)))
grep_count "$regex_x1" $((0+(24*i)))
grep_count "$regex_y0" $((24+(24*i)))
grep_count "$regex_y1" $((0+(24*i)))

try fdb-write data.xxxx.1.grib
try fdb-list --all --minimum-keys="" --porcelain --full > out
count $((72+(i*96)))
try fdb-list --all --minimum-keys="" --porcelain --full | tee out
line_count $((72+(i*96)))
grep_count "$regex_x0" $((24+(24*i)))
grep_count "$regex_x1" $((24+(24*i)))
grep_count "$regex_y0" $((24+(24*i)))
grep_count "$regex_y1" $((0+(24*i)))

try fdb-write data.xxxy.1.grib
try fdb-list --all --minimum-keys="" --porcelain --full > out
count $((96+(i*96)))
try fdb-list --all --minimum-keys="" --porcelain --full | tee out
line_count $((96+(i*96)))
grep_count "$regex_x0" $((24+(24*i)))
grep_count "$regex_x1" $((24+(24*i)))
grep_count "$regex_y0" $((24+(24*i)))
Expand Down
34 changes: 16 additions & 18 deletions tests/fdb/tools/list/list_depth.sh.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,11 @@

set -ux

yell() { printf "$(basename "$0"): \033[0;31m%s\033[0m\\n" "$*" >&2; }
yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; }
die() { yell "$*"; exit 1; }
try() { "$@" || die "Errored HERE => '$*'"; }

function request() {
echo "RETRIEVE,CLASS=OD,TYPE=CF,STREAM=ENFO,EXPVER=0001,LEVTYPE=PL,DATE=20201106,TIME=1200,STEP=00,DOMAIN=G,$1" > req;
}
########################################################################################################################

export PATH=@CMAKE_BINARY_DIR@/bin:$PATH
export FDB5_CONFIG_FILE="local.yaml"
Expand All @@ -25,8 +23,8 @@ echo "running test '$test_name' on $(hostname)"

try cd $bin_dir

rm -rf $test_name || true
mkdir -p $test_name/localroot
try rm -rf $test_name
try mkdir -p $test_name/localroot

try cd $test_name

Expand All @@ -52,9 +50,8 @@ done
try fdb-write "$test_name.grib"

########################################################################################################################
# test all

out=$(fdb-list --all --minimum-keys= --porcelain)
echo "Test: all"
exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=0,param=166}
{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166}
{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=6,param=166}
Expand All @@ -63,33 +60,34 @@ exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc
{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166}
{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166}
{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}"
out=$(try fdb-list --all --minimum-keys= --porcelain)
try test "$exp" = "$out"

# test date depth=1
out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=1)
echo "Test: date depth=1"
exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}"
out=$(try fdb-list date=20201102 --minimum-keys="" --porcelain --depth=1)
try test "$exp" = "$out"

# test date depth=2
out=$(fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2)
echo "Test: date depth=2"
exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}
{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}"
out=$(try fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2)
try test "$exp" = "$out"

# test cf depth=1
out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=1)
echo "Test: cf depth=1"
exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}"
out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=1)
try test "$exp" = "$out"

# test cf depth=2
out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=2)
echo "Test: cf depth=2"
exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}"
out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=2)
try test "$exp" = "$out"

# test cf depth=3
out=$(fdb-list type=an --minimum-keys="" --porcelain --depth=3)
echo "Test: cf depth=3"
exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166}
{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166}
{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166}
{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}"
out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=3)
try test "$exp" = "$out"
Loading

0 comments on commit 13eeff9

Please sign in to comment.