runt.toml

ver = "0.4.1"

##### Compilation Tests ######
## Test each pass in isolation. Gets the pass flags from a comment on the first line of the file
[[tests]]
name = "[core] passes"
paths = ["tests/passes/**/*.futil"]
# gets the pass flags a comment on the first line of the test file
cmd = """
flags="$(head -n 1 {} | cut -c 3-) -m file"
./target/debug/calyx {} $flags -l . \
  | sed 's/extern \".*\\(calyx\\/.*\\)\"/extern \"<ROOT>\\/\\1\"/'
"""

[[tests]]
name = "[core] parsing"
# Round-tripping from the compiler should not change anything.
paths = ["./tests/parsing/*.expect"]
cmd = """
./target/debug/calyx {} -m file -p none -l .
"""

[[tests]]
name = "[core] import"
# Round-tripping from the compiler should not change anything.
paths = ["./tests/import/a.futil"]
cmd = """
./target/debug/calyx {} -p none -l . \
  | sed 's/extern \".*\\(calyx\\/.*\\)\"/extern \"<ROOT>\\/\\1\"/'
"""

## Tests the error messages generated by the compiler. Runs passes for error
## checking.
[[tests]]
name = "[core] errors"
paths = [
  "tests/errors/*.futil",
  "tests/errors/papercut/*.futil",
  "tests/errors/parser/*.futil",
]
cmd = """
./target/debug/calyx {} -p well-formed -p papercut -p synthesis-papercut -l . -m file
"""

[[tests]]
name = "[core] futil examples"
paths = ["examples/futil/*.futil"]
cmd = """
./target/debug/calyx {} -m file -l .
"""

##### Backend Tests #####
[[tests]]
name = "[core] backends"
paths = ["tests/backend/**/*.futil"]
cmd = """
flags=$(head -n 1 {} | cut -c 3-)
./target/debug/calyx {} $flags -l .
"""

[[tests]]
name = "yxi tool"
paths = ["yxi/tests/yxi-tool/*.futil"]
cmd = """
./target/debug/yxi {}
"""

[[tests]]
name = "fud2 yxi invocation"
paths = ["yxi/tests/ref-mems-vec-add.futil"]
cmd = """
fud2 {} --from calyx --to yxi
"""

##### Frontend Tests #####
[[tests]]
name = "[frontend] dahlia"
paths = ["tests/frontend/dahlia/*.fuse"]
cmd = """
fud e {} --to calyx -q \
          -s calyx.exec './target/debug/calyx' \
        | tail -n+2
"""

[[tests]]
name = "[frontend] systolic array"
paths = ["tests/frontend/systolic/array-*.systolic"]
cmd = """
./frontends/systolic-lang/gen-systolic.py {}
"""

[[tests]]
name = "[frontend] NTT pipeline generation"
paths = ["tests/frontend/ntt-pipeline/*.txt"]
cmd = """
fud e --from ntt {} --to calyx -q
"""

[[tests]]
name = "[frontend] relay"
paths = ["tests/frontend/relay/*.relay"]
cmd = "python3 frontends/relay/relay_visitor.py {}"

[[tests]]
name = "[frontend] exp"
paths = ["tests/frontend/exp/*.txt"]
cmd = "python3 calyx-py/calyx/gen_exp.py {}"

[[tests]]
name = "[frontend] queues correctness"
paths = [
    "frontends/queues/tests/*.py",
    "frontends/queues/tests/round_robin/*.py",
    "frontends/queues/tests/strict/*.py",
    "frontends/queues/tests/binheap/*.py"
]
cmd = """
name=$(basename {} .py) &&
dir=$(dirname {}) &&
python3 {} 20000 --keepgoing |\
 fud e --from calyx --to jq \
       --through verilog \
       --through dat \
       -s verilog.data "$dir/$name.data" \
       -s jq.expr ".memories" \
       -q
"""

[[tests]]
name = "[frontend] queues sdn no-promote correctness"
paths = ["frontends/queues/tests/sdn.py"]
cmd = """
name=$(basename {} .py) &&
dir=$(dirname {}) &&
python3 {} 20000 --keepgoing |\
 fud e --from calyx --to jq \
       --through verilog \
       --through dat \
       -s verilog.data "$dir/$name.data" \
       -s calyx.flags ' -d static-promotion' \
       -s jq.expr ".memories" \
       -q
"""

# A stanza for "non-static with promotion" is unnecessary.
# Such a stanza would have invoked `sdn.py` with no special flags.
# The catch-all queues stanza captures that case.

[[tests]]
name = "[frontend] queues sdn static no promote correctness"
paths = ["frontends/queues/tests/sdn.py"]
cmd = """
name=$(basename {} .py) &&
dir=$(dirname {}) &&
python3 {} 20000 --static --keepgoing |\
 fud e --from calyx --to jq \
       --through verilog \
       --through dat \
       -s verilog.data "$dir/$name.data" \
       -s calyx.flags ' -d well-formed -d static-promotion' \
       -s jq.expr ".memories" \
       -q
"""

[[tests]]
name = "[frontend] queues sdn static promote correctness"
paths = ["frontends/queues/tests/sdn.py"]
cmd = """
name=$(basename {} .py) &&
dir=$(dirname {}) &&
python3 {} 20000 --static --keepgoing |\
 fud e --from calyx --to jq \
       --through verilog \
       --through dat \
       -s verilog.data "$dir/$name.data" \
       -s calyx.flags ' -d well-formed' \
       -s jq.expr ".memories" \
       -q
"""

[[tests]]
name = "[calyx-py] builder"
paths = ["calyx-py/test/*.py"]
cmd = """
python3 {}
"""

[[tests]]
name = "[calyx-py] correctness"
paths = ["calyx-py/test/correctness/*.py"]
cmd = """
name=$(basename {} .py) &&
dir=$(dirname {}) &&
python3 {} |\
 fud e --from calyx --to jq \
       --through verilog \
       --through dat \
       -s verilog.data "$dir/$name.data" \
       -s jq.expr ".memories" \
       -q
"""

##### Correctness Tests #####
## Tests that ensure that individual control
## constructs have correct behavior when compiled.
[[tests]]
name = "correctness dynamic"
paths = [
  "tests/correctness/*.futil",
  "tests/correctness/ref-cells/*.futil",
  "tests/correctness/sync/*.futil",
  "tests/correctness/static-interface/*.futil",
]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s verilog.data {}.data \
         -s calyx.exec './target/debug/calyx' \
         -s verilog.cycle_limit 500 \
         -s jq.expr ".memories" \
         {} -q
"""
timeout = 120

[[tests]]
name = "correctness dynamic one-hot encoding"
paths = [
  "tests/correctness/*.futil",
  "tests/correctness/ref-cells/*.futil",
  "tests/correctness/sync/*.futil",
  "tests/correctness/static-interface/*.futil",
]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s calyx.exec './target/debug/calyx' \
         -s calyx.flags '-x tdcc:one-hot-cutoff=500 -d static-promotion' \
         -s verilog.cycle_limit 500 \
         -s verilog.data {}.data \
         -s jq.expr ".memories" \
         {} -q
"""
timeout = 120

[[tests]]
name = "correctness dynamic fsm register duplication"
paths = [
  "tests/correctness/*.futil",
  "tests/correctness/ref-cells/*.futil",
  "tests/correctness/sync/*.futil",
  "tests/correctness/static-interface/*.futil",
]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s calyx.exec './target/debug/calyx' \
         -s calyx.flags '-x tdcc:duplicate-cutoff=0 -d static-promotion' \
         -s verilog.cycle_limit 500 \
         -s verilog.data {}.data \
         -s jq.expr ".memories" \
         {} -q
"""
timeout = 120

[[tests]]
name = "correctness static timing"
paths = [
  "tests/correctness/*.futil",
  "tests/correctness/ref-cells/*.futil",
  "tests/correctness/static-interface/*.futil",
]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s calyx.exec './target/debug/calyx' \
         -s verilog.cycle_limit 500 \
         -s verilog.data {}.data \
         -s jq.expr ".memories" \
         {} -q
"""
timeout = 120

[[tests]]
name = "correctness static timing one-hot encoding"
paths = ["tests/correctness/static-interface/*.futil"]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s calyx.exec './target/debug/calyx' \
         -s calyx.flags '-x static-fsm-opts:one-hot-cutoff=500' \
         -s verilog.cycle_limit 500 \
         -s verilog.data {}.data \
         -s jq.expr ".memories" \
         {} -q
"""
timeout = 120

# Variants of the above but for `--nested` (non-ANF Verilog emission) mode.
[[tests]]
name = "correctness nested"
paths = ["tests/correctness/*.futil"]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s verilog.data {}.data \
         -s calyx.exec './target/debug/calyx' \
         -s calyx.flags ' --nested' \
         -s verilog.cycle_limit 500 \
         -s jq.expr ".memories" \
         {} -q
"""
timeout = 120

# Tests to ensure that static compilation maintains its guarantees
# See: https://github.com/calyxir/calyx/discussions/1331 & https://github.com/calyxir/calyx/issues/1344
[[tests]]
name = "correctness static control"
paths = [
  "tests/correctness/static-control/*.futil",
  "tests/correctness/group-static-promotion/*.futil",
]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s calyx.exec './target/debug/calyx' \
         -s calyx.flags '-p all -d group2invoke' \
         -s verilog.cycle_limit 500 \
         -s verilog.data {}.data \
         {} -q
"""
timeout = 120

# Tests to ensure static compilation maintains guarantees for one-hot encodings
[[tests]]
name = "correctness static control, one-hot fsms"
paths = [
  "tests/correctness/static-control/*.futil",
  "tests/correctness/group-static-promotion/*.futil",
]
cmd = """
fud exec --from calyx --to jq \
         --through verilog \
         --through dat \
         -s calyx.exec './target/debug/calyx' \
         -s calyx.flags '-p all -d group2invoke -x static-fsm-opts:one-hot-cutoff=500' \
         -s verilog.cycle_limit 500 \
         -s verilog.data {}.data \
         {} -q
"""
timeout = 120

[[tests]]
name = "numeric types correctness and parsing"
paths = [
  "tests/correctness/numeric-types/parsing/*.futil",
  "tests/correctness/numeric-types/bitnum/*.futil",
  "tests/correctness/numeric-types/fixed-point/*.futil",
]
cmd = """
fud exec --from calyx --to jq \
         --through dat \
         --through verilog \
         -s verilog.data {}.data \
         -s jq.expr ".memories" \
         {} -q
"""

[[tests]]
name = "correctness test of static islands without static promotion"
paths = ["tests/correctness/static-islands/*.futil"]
cmd = """
fud exec --from calyx --to jq \
         --through dat \
         --through verilog \
         -s verilog.data {}.data \
         -s calyx.flags "-d static-promotion" \
         -s jq.expr ".memories" \
         {} -q
"""

[[tests]]
name = "[frontend] tcam testing"
paths = ["tests/correctness/tcam/*.futil"]
cmd = """
fud exec --from calyx --to jq \
      --through verilog \
      --through dat \
      -s verilog.data {}.data \
      -s jq.expr ".memories" \
      {} -q
"""

[[tests]]
name = "[frontend] systolic array PE correctness"
paths = ["tests/correctness/systolic/pe/*.systolic"]
cmd = """
fud e --from systolic --to jq \
      --through verilog \
      --through vcd_json \
      -s verilog.data {}.data \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      -s jq.file {}.jq \
      {} -q
"""

[[tests]]
name = "[frontend] systolic array mmult static correctness"
paths = ["tests/correctness/systolic/mmult-inputs/*.data"]
cmd = """
fud e --from systolic --to jq \
      --through verilog \
      --through dat \
      -s verilog.data {} \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      -s jq.expr ".memories" \
      {}_static.systolic -q
"""
expect_dir = "tests/correctness/systolic/mmult-expect"

[[tests]]
name = "[frontend] systolic array mmult dynamic correctness"
paths = ["tests/correctness/systolic/mmult-inputs/*.data"]
cmd = """
fud e --from systolic --to jq \
      --through verilog \
      --through dat \
      -s verilog.data {} \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      -s jq.expr ".memories" \
      {}_dynamic.systolic -q
"""
expect_dir = "tests/correctness/systolic/mmult-expect"

[[tests]]
name = "[correctness] `@fast` attribute should not change program behavior"
paths = ["tests/correctness/fast-attr/valid/*.futil"]
cmd = """
fud exec --from calyx --to jq \
         --through dat \
         --through icarus-verilog \
         -s calyx.flags "-p validate -p compile -p lower -d static-promotion" \
         -s verilog.data {}.data \
         -s jq.expr "." \
         {} -q
"""

[[tests]]
name = "[correctness] `@fast` attribute should reject unoptimizable sequences"
paths = ["tests/correctness/fast-attr/invalid/*.futil"]
cmd = """
fud exec --from calyx --to jq \
         --through dat \
         --through icarus-verilog \
         -s calyx.flags "-p validate -p compile -p lower -d static-promotion" \
         -s verilog.data {}.data \
         -s jq.expr "." \
         {} -q
"""

[[tests]]
name = "[frontend] systolic array relu static correctness"
paths = ["tests/correctness/systolic/relu-inputs/*.data"]
cmd = """
fud e --from systolic --to jq \
      --through verilog \
      --through dat \
      -s verilog.data {} \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      -s jq.expr ".memories" \
      {}_static.systolic -q
"""
expect_dir = "tests/correctness/systolic/relu-expect"

[[tests]]
name = "[frontend] systolic array relu dynamic correctness"
paths = ["tests/correctness/systolic/relu-inputs/*.data"]
cmd = """
fud e --from systolic --to jq \
      --through verilog \
      --through dat \
      -s verilog.data {} \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      -s jq.expr ".memories" \
      {}_dynamic.systolic -q
"""
expect_dir = "tests/correctness/systolic/relu-expect"

[[tests]]
name = "[frontend] systolic array leaky relu correctness"
paths = ["tests/correctness/systolic/leaky-relu/*.systolic"]
cmd = """
fud e --from systolic --to dat \
      --through verilog \
      -s verilog.data {}.data \
      -s calyx.exec './target/debug/calyx' \
      -s calyx.flags "-d well-formed" \
      {} -q
"""

[[tests]]
name = "[frontend] NTT pipeline correctness"
paths = ["tests/correctness/ntt-pipeline/*.txt"]
cmd = """
fud e --from ntt {} --through verilog --to dat -s verilog.data {}.data -q
"""

[[tests]]
name = "exp correctness"
paths = ["tests/correctness/exp/*.txt"]
cmd = """
python3 calyx-py/calyx/gen_exp.py {} |\
fud e --from calyx --to dat --through verilog -s verilog.data {}.data -q
"""

[[tests]]
name = "[frontend] mrxl correctness"
paths = ["frontends/mrxl/test/*.mrxl"]
expect_dir = "frontends/mrxl/test/expect/simulate_out/"
cmd = """
fud e --from mrxl {} \
      --to jq \
      --through verilog \
      --through dat \
      -s mrxl.data {}.data \
      -s jq.expr ".memories" \
      -q
"""

[[tests]]
name = "[frontend] relay correctness"
paths = ["tests/correctness/relay/*.relay"]
cmd = """
fud e -q {} --from relay --to jq \
            --through verilog \
            --through dat \
            -s verilog.data {}.data \
            -s jq.expr ".memories"
"""

## Tests errors that occur at runtime
[[tests]]
name = "runtime errors"
paths = ["tests/errors/runtime/*.futil"]
cmd = """
fud exec -s calyx.exec './target/debug/calyx' \
         -s verilog.cycle_limit 500 \
         --through verilog \
         {} --from calyx --to dat -q 2>&1 | \
         grep '%Error' | sed 's/%Error: [^:]*:[^:]*:/%Error:/'
"""

##### Examples #####

[[tests]]
name = "[examples] language tutorial"
paths = ["examples/tutorial/*.futil"]
cmd = """
fud e {} -s verilog.cycle_limit 500 \
         --through verilog \
         -s verilog.data examples/tutorial/data.json \
         --from calyx --to dat -q
"""

[[tests]]
name = "[examples] sync primitives"
paths = ["examples/sync/*.futil"]
cmd = """
fud e {} -s verilog.cycle_limit 500 \
         --through verilog \
         -s verilog.data {}.data \
         --from calyx --to dat -q
"""

[[tests]]
name = "[examples] memory by reference tutorial"
paths = ["examples/futil/memory-by-reference/*.futil"]
cmd = """
fud e {} -s verilog.cycle_limit 500 \
         --through verilog \
         -s verilog.data {}.data \
         --from calyx --to dat -q
"""

[[tests]]
name = "[examples] dahlia examples"
paths = ["examples/dahlia/*.fuse"]
cmd = """
fud e {} -s verilog.cycle_limit 500 \
         --through verilog \
         -s verilog.data {}.data \
         --to dat -q
"""

##### Calyx AXI tests #####
[[tests]]
name = "calyx-py read-write-compute AXI wrapper generation"
paths = ["yxi/tests/axi/read-compute-write/*.yxi"]
cmd = """
python3 yxi/axi-calyx/axi-generator.py {}
"""

[[tests]]
name = "calyx-py dynamic AXI wrapper generation"
paths = ["yxi/tests/axi/dynamic/*.yxi"]
cmd = """
python3 yxi/axi-calyx/dynamic-axi-generator.py {}
"""

#Ignore fud2 stderr for now due to ninja nondeterminism
[[tests]]
name = "fud2 AXI wrapper invocation"
paths = ["yxi/tests/axi/*/*-mem-vec-add.futil"]
cmd = """
dynamic=$(head -n 2 {} | tail -n 1 | cut -c 4-)
fud2 {} --to calyx --through axi-wrapped --set $dynamic 2> /dev/null
"""

[[tests]]
name = "fud2 AXI-wrapped to verilog"
paths = ["yxi/tests/axi/*/*-mem-vec-add-axi-wrapped.futil"]
cmd = """
fud2 {} --from calyx --to verilog-noverify 2> /dev/null
"""

[[tests]]
name = "fud2 cocotb AXI-wrapped xecution"
paths = ["yxi/tests/axi/*/*-mem-vec-add-verilog.v"]
cmd = """
fud2 {} --from verilog-noverify --to cocotb-axi --set sim.data=yxi/tests/axi/vectorized-add.data 2> /dev/null
"""

##### Xilinx Tests ######
[[tests]]
name = "AXI wrapper generation"
paths = ["tests/xilinx/*.futil"]
expect_dir = "tests/xilinx/gen-verilog"
cmd = """
target/debug/calyx {} -b xilinx
"""

[[tests]]
name = "kernel.xml generation"
paths = ["tests/xilinx/*.futil"]
expect_dir = "tests/xilinx/gen-xml"
cmd = """
target/debug/calyx {} -b xilinx-xml
"""

[[tests]]
name = "kernel.xml generation from yxi"
paths = ["yxi/tests/axi/xml/*.yxi"]
cmd = """
python3 yxi/xml/xml_generator.py {}
"""


[[tests]]
name = "Cocotb correctness tests"
paths = [
  "tests/xilinx/cocotb/dot-product/",
  "tests/xilinx/cocotb/vectorized-add/",
]
cmd = """
mkdir -p {}/hdl && \
dir_name=$(basename {}) && \
fud e -q --from calyx {}/${dir_name}.futil --to synth-verilog -o {}/hdl/main.sv &&\
  fud e -q --from calyx {}/${dir_name}.futil --to axi-wrapper -o {}/hdl/toplevel.v

cd {}/../ && \
make --silent -B TEST_PATH={} COCOTB_LOG_LEVEL=CRITICAL |\
    grep -E '^Output:' |\
    sed 's/Output://g' &&\
  make clean && \
  rm -f results.xml

"""

### Profiler tests
[[tests]]
name = "profiler"
paths = ["tests/profiler/*.futil"]
cmd = """
bash tools/profiler/get-profile-counts-info.sh {} {}.data STDOUT -d
"""