[prakriya] Improve snapshot tests and fix bugs

Snapshot tests are much faster, and disk usage is a fraction of what it
was previously.

Changes made:

- Merge snapshot test binaries into a single example binary.
- Switch data format from CSV to compressed MessagePack.
- Move various binary dependencies to dev-dependencies.
- Add tests for sya-Satf, sya-SAnac (kartari + karmani), and yak-SAnac.
- Improve threading distribution.

+--------+-------------------+----------+------------+
|        | create_test_files | test_all | disk usage |
+--------+-------------------+----------+------------+
| Before |           339.73s |  101.87s |      6800M |
| After  |            86.62s |   45.42s |        90M |
+--------+-------------------+----------+------------+

In addition, make some small bug fixes:

- Add 6.4.50
- Fix SK 2605
- Fix bug with saMstti / santti (#122)
- Fix bug with apusPurat (#128)
- Fix bug with lAyayati (#146)

vidyut-prakriya/.gitignore

@@ -1,3 +1,4 @@
 www/static/data/
 www/static/wasm/
 *.log
+*.msgpack

vidyut-prakriya/Cargo.toml

@@ -12,21 +12,23 @@ edition = "2021"
 rust-version = "1.68"
 
 [dependencies]
-clap = { version = "4.0.12", features = ["derive"] }
-csv = "1.1.6"
 enumset = { version = "1.1.3", features = ["serde"] }
 lazy_static = "1.4.0"
+rustc-hash = "2.0.0"
 serde = { version = "1.0.150", features = ["derive"] }
-sha2 = "0.10.6"
-sha256 = "1.1.1"
-rayon = "1.6.1"
 wasm-bindgen = { version = "0.2", features = ["serde-serialize"]}
 serde-wasm-bindgen = "0.4"
 console_error_panic_hook = "0.1.7"
-rustc-hash = "2.0.0"
 
 [dev-dependencies]
+clap = { version = "4.0.12", features = ["derive"] }
 criterion = "0.5.1"
+csv = "1.3.1"
+flate2 = "1.0.35"
+rayon = "1.6.1"
+rmp-serde = "1.3.0"
+sha2 = "0.10.6"
+sha256 = "1.1.1"
 test_utils = { path = "test_utils" }
 vidyut-lipi = { path = "../vidyut-lipi" }
 

vidyut-prakriya/Makefile

@@ -1,3 +1,11 @@
+# Build
+# -----
+
+# Verifies that everything builds.
+build_all:
+	cargo build --examples
+
+
 # Unit tests
 # ~~~~~~~~~~
 
@@ -15,8 +23,7 @@ coverage:
 
 # Generates all tinantas, krdantas, and dhatus supported by the program and writes them to disk.
 create_test_files:
-	cargo build --release --bin create_test_files
-	../target/release/create_test_files --output-dir test-files
+	cargo run --release --example snapshot_tests create --output-dir test-files
 
 # Runs a full evaluation over all forms generated by vidyut-prakriya. `hash` is
 # the SHA-256 hash of the test file. We use `hash` to verify test file
@@ -27,65 +34,46 @@ create_test_files:
 test_all: test_tinantas test_krdantas test_dhatus
 
 test_tinantas:
-	cargo build --release --bin test_results
-	../target/release/test_results \
-		--data-type tinanta \
-		--test-cases test-files/tinantas-basic.csv \
-	  --hash "8ae177857fb0ba870a6269ec6d96a78ee7e3d6ae0586f69e072b29ccc9108562"
-	../target/release/test_results \
-		--data-type tinanta \
-  	--test-cases test-files/tinantas-nic.csv \
-    --hash "1cc63c02a628e8ed843c455c90dd327f25c7c824fea32e8642851b0011e47666"
-	../target/release/test_results \
-		--data-type tinanta \
-		--test-cases test-files/tinantas-san.csv \
-		--hash "69c5de2266c7a8911fbea09fbef85951c94da818f7994d560ce62de92dd18074"
-	../target/release/test_results \
-		--data-type tinanta \
-		--test-cases test-files/tinantas-yan.csv \
-		--hash "436443c42c36e075bb866088b92456a8fedbe8c5f63686cae72d9f04897402ef"
-	../target/release/test_results \
-		--data-type tinanta \
-		--test-cases test-files/tinantas-yan-luk.csv \
-		--hash "bf4fbfd3e775057628656ce3af7760a2f80b69e2a410086465d616330e7041a7"
-	../target/release/test_results \
-		--data-type tinanta \
-		--test-cases test-files/tinantas-san-nic.csv \
-		--hash "4b7a5b83786a3ef53dec0bbf580a224f5161bf2f3dc2f2e8a917ba231e25e225"
-	../target/release/test_results \
-		--data-type tinanta \
-		--test-cases test-files/tinantas-nic-san.csv \
-		--hash "fb647997a7f2956a45deeb7400d8ed11359448a0edc43c16725ba597855db768"
+	cargo run --release --example snapshot_tests validate \
+		--test-type tinantas \
+		-f test-files/tinantas-basic.msgpack \
+		-h "1ca53f0f7612728943699d20b5e42d80ca340ed330102cdede1e2e3b8af9e973" \
+		-f test-files/tinantas-nic.msgpack \
+		-h "a8b186ff62fe20a08e147537614bbb3220fbfd8d2fce356514e5e595d1e579fb" \
+		-f test-files/tinantas-san.msgpack \
+		-h "6a5180111ba24c8a6cbdb9257990438c816515b5102193449857ab20bd07164e" \
+		-f test-files/tinantas-yan.msgpack \
+		-h "0ba293917913f2aed9423ea3125071c56426ca2fc9e06a2632f27297de4b0139" \
+		-f test-files/tinantas-yan-luk.msgpack \
+		-h "871c631461aa2ab1ded884de5bdf12464879f5da46c660c3076c333a3a90fd7f" \
+		-f test-files/tinantas-san-nic.msgpack \
+		-h "66b82321c244f4369422b06c2db9254d61a680eceeb6e9c0e8b8ac24a274b3c8" \
+		-f test-files/tinantas-nic-san.msgpack \
+		-h "c84fc7a9cfce47561b5075f974f5fae4a2820549336c344a97978f719c01e409"
 
 test_krdantas:
-	cargo build --release --bin test_results
-	../target/release/test_results \
-		--data-type krdanta \
-		--test-cases test-files/krdantas-basic.csv \
-	  --hash "6160c100552db7946294e12054e57111e36e766d726dd0268b828eeb14056369"
-	../target/release/test_results \
-		--data-type krdanta \
-		--test-cases test-files/krdantas-nic.csv \
-	  --hash "d7579416e7c2041f50c8502e3f48610e8f874625e39b215db782240829baf63e"
-	../target/release/test_results \
-		--data-type krdanta \
-		--test-cases test-files/krdantas-san.csv \
-	  --hash "66d71a1de6fc1803edb7e65ee1332e2432aa6f8f29ec9055a86c646307c7207f"
-	../target/release/test_results \
-		--data-type krdanta \
-		--test-cases test-files/krdantas-yan.csv \
-	  --hash "5512ddc37a4ead2d0f983875f3c705c0237d2d482a24dcff650e8756c45f0f0e"
-	../target/release/test_results \
-		--data-type krdanta \
-		--test-cases test-files/krdantas-yan-luk.csv \
-	  --hash "ad5d53cf24b411e0b9ffbf7a95ee36113d76551ddef0ce06128fb44e436570a1"
+	cargo run --release --example snapshot_tests validate \
+		--test-type krdantas \
+		-f test-files/krdantas-basic.msgpack \
+		-h "07aa1d71eef9f7c557de5ffa07dafcfe2b1836fc09779421794055bd4143af90" \
+		-f test-files/krdantas-nic.msgpack \
+		-h "bdbadc06d3d94831c3eb1d8190286e289c10c3c55239b7c740bbea86e64a7062" \
+		-f test-files/krdantas-san.msgpack \
+		-h "7f7195e09f1e92e1af194c9fd01f08adf20d209d8e2dacd313e49de444f28eaf" \
+		-f test-files/krdantas-yan.msgpack \
+		-h "18f85a147517c8481d6355175aae5e6f0d144c4d1a029f2b2a9f49276d562b67" \
+		-f test-files/krdantas-yan-luk.msgpack \
+		-h "8bb265c99bd29ead515aef14e031c4fb9230b3abb8f2f2dc1387e48a8525a83a" \
+		-f test-files/krdantas-san-nic.msgpack \
+		-h "2db472ab2afeca38c40be55e8fe261441130b70fe0165696071cbe2ef9332a12" \
+		-f test-files/krdantas-nic-san.msgpack \
+		-h "ddefad731e0393fdf9896fe42a5187b7a060082e9043096f8969f463c888da4f"
 
 test_dhatus:
-	cargo build --release --bin test_results
-	../target/release/test_results \
-		--data-type dhatu \
-		--test-cases test-files/dhatus.csv \
-	  --hash "aef40ba22893ed6d6c7fd7c126b93435faa9418491002b1dc8ae895c7a6e28e8"
+	cargo run --release --example snapshot_tests validate \
+		--test-type dhatus \
+		-f test-files/dhatus.msgpack \
+		-h "b76f6643625ca659b4fb7e7dd8534d9204237ec20c512943a4d87441ec938dae"
 
 
 test_subantas:

vidyut-prakriya/data/varttikas.tsv

@@ -19,6 +19,7 @@
 3.1.16.1	PenAcceti vaktavyam
 3.1.17.1	sudinadurdinanIhareByaSceti vaktavyam
 3.1.17.2	awAwwASIkAkowApowAsowApruzwApluzwAgrahaRaM kartavyam
+3.1.21.1	halikalyoradantatvaM nipAtyate
 3.1.22.1	sUcisUtramUtryawyartyaSUrRotInAM grahaRaM
 3.1.25.1	arTavedasatyAnAmApugvaktavyaH
 3.1.34.1	sibbahulaM Candasi RidvaktavyaH
@@ -84,6 +85,7 @@
 7.1.26.1	ekatarAtpratizeDo vaktavyaH
 7.1.59.1	Se tfmPAdInAmupasaNKyAnaM kartavyam
 7.1.90.1	oto Riditi vAcyam
+7.2.13.1	kfYo'suwkasyeti vaktavyam
 7.2.49.1	tanipatidaridrARAmupasaNKyAnam
 7.2.68.1	dfSeSceti vaktavyam
 7.3.34.1	anAcamikamivamInAm iti vaktavyam