Rewrite lexer and parser (#196)

Cargo.toml

@@ -1,7 +1,6 @@
 [workspace]
 members = [
     "jaq-syn",
-    "jaq-parse",
     "jaq-interpret",
     "jaq-core",
     "jaq-std",

jaq-core/Cargo.toml

@@ -29,5 +29,5 @@ base64 = { version = "0.22", optional = true }
 urlencoding = { version = "2.1.3", optional = true }
 
 [dev-dependencies]
-jaq-parse = { version = "1.0.0", path = "../jaq-parse" }
+jaq-syn = { version = "1.6.0", path = "../jaq-syn" }
 serde_json = "1.0"

jaq-core/tests/common/mod.rs

@@ -4,9 +4,10 @@ fn yields(x: jaq_interpret::Val, f: &str, ys: impl Iterator<Item = jaq_interpret
     let mut ctx = jaq_interpret::ParseCtx::new(Vec::new());
     ctx.insert_natives(jaq_core::core());
 
-    let (f, errs) = jaq_parse::parse(f, jaq_parse::main());
-    assert!(errs.is_empty());
-    ctx.yields(x, f.unwrap(), ys)
+    let f = jaq_syn::parse(f, |p| p.module(|p| p.term()))
+        .unwrap()
+        .conv(f);
+    ctx.yields(x, f, ys)
 }
 
 pub fn fail(x: Value, f: &str, err: jaq_interpret::Error) {

jaq-interpret/Cargo.toml

@@ -23,6 +23,3 @@ hifijson = { version = "0.2.0", optional = true }
 indexmap = "2.0"
 once_cell = "1.16.0"
 serde_json = { version = "1.0.81", optional = true }
-
-[dev-dependencies]
-jaq-parse = { version = "1.0.0", path = "../jaq-parse" }

jaq-interpret/src/lib.rs

@@ -22,11 +22,10 @@
 //! let mut defs = ParseCtx::new(Vec::new());
 //!
 //! // parse the filter
-//! let (f, errs) = jaq_parse::parse(filter, jaq_parse::main());
-//! assert_eq!(errs, Vec::new());
+//! let f = jaq_syn::parse(filter, |p| p.module(|p| p.term())).unwrap().conv(filter);
 //!
 //! // compile the filter in the context of the given definitions
-//! let f = defs.compile(f.unwrap());
+//! let f = defs.compile(f);
 //! assert!(defs.errs.is_empty());
 //!
 //! let inputs = RcIter::new(core::iter::empty());

jaq-interpret/tests/common/mod.rs

@@ -2,9 +2,10 @@ use serde_json::Value;
 
 fn yields(x: jaq_interpret::Val, f: &str, ys: impl Iterator<Item = jaq_interpret::ValR>) {
     let mut ctx = jaq_interpret::ParseCtx::new(Vec::new());
-    let (f, errs) = jaq_parse::parse(f, jaq_parse::main());
-    assert!(errs.is_empty());
-    ctx.yields(x, f.unwrap(), ys)
+    let f = jaq_syn::parse(f, |p| p.module(|p| p.term()))
+        .unwrap()
+        .conv(f);
+    ctx.yields(x, f, ys)
 }
 
 pub fn fail(x: Value, f: &str, err: jaq_interpret::Error) {

jaq-interpret/tests/path.rs

@@ -42,7 +42,6 @@ fn index_access() {
 fn iter_access() {
     gives(json!([0, 1, 2]), ".[]", [json!(0), json!(1), json!(2)]);
     gives(json!({"a": [1, 2]}), ".a[]", [json!(1), json!(2)]);
-    gives(json!({"a": [1, 2]}), ".a.[]", [json!(1), json!(2)]);
     gives(json!({"a": 1, "b": 2}), ".[]", [json!(1), json!(2)]);
     // TODO: correct this
     //gives(json!({"b": 2, "a": 1}), ".[]", [json!(2), json!(1)]);
@@ -74,6 +73,9 @@ fn iter_assign() {
     );
 }
 
+yields!(index_keyword, r#"{"if": 0} | .if"#, 0);
+yields!(obj_keyword, "{if: 0} | .if", 0);
+
 yields!(key_update1, "{} | .a  |= .+1", json!({"a": 1}));
 yields!(key_update2, "{} | .a? |= .+1", json!({"a": 1}));
 

jaq-interpret/tests/tests.rs

@@ -20,9 +20,9 @@ yields!(cartesian_arith, "[(1,2) * (3,4)]", [3, 4, 6, 8]);
 #[test]
 fn add() {
     give(json!(1), ". + 2", json!(3));
-    give(json!(1.0), ". + 2.", json!(3.0));
+    give(json!(1.0), ". + 2.0", json!(3.0));
     give(json!(1), "2.0 + .", json!(3.0));
-    give(json!(null), "1.e1 + 2.1e2", json!(220.0));
+    give(json!(null), "1.0e1 + 2.1e2", json!(220.0));
 
     give(json!("Hello "), ". + \"world\"", json!("Hello world"));
     give(json!([1, 2]), ". + [3, 4]", json!([1, 2, 3, 4]));
@@ -48,7 +48,7 @@ yields!(sub_arr, "[1, 2, 3] - [2, 3, 4]", json!([1]));
 #[test]
 fn mul() {
     give(json!(1), ". * 2", json!(2));
-    give(json!(1.0), ". * 2.", json!(2.0));
+    give(json!(1.0), ". * 2.0", json!(2.0));
     give(json!(1), "2.0 * .", json!(2.0));
 
     give(json!("Hello"), "2 * .", json!("HelloHello"));
@@ -117,6 +117,27 @@ fn precedence() {
     give(json!(null), "2 * 3 + 1", json!(7));
 }
 
+// these tests use the trick that `try t catch c` is valid syntax only for atomic terms `t`
+// TODO for v2.0
+//yields!(atomic_def, "try def x: 1; x + x catch 0", 2);
+yields!(atomic_neg, "try - 1 catch 0", -1);
+yields!(atomic_if, "try if 0 then 1 end catch 2", 1);
+yields!(atomic_try, "try try 0[0] catch 1 catch 2", 1);
+yields!(atomic_fold, "try reduce [][] as $x (0; 0) catch 1", 0);
+yields!(atomic_var, "0 as $x | try $x catch 1", 0);
+yields!(atomic_call, "def x: 0; try x catch 1", 0);
+yields!(atomic_str1, r#"try "" catch 1"#, "");
+yields!(atomic_str2, r#"def @f: .; try @f "" catch 1"#, "");
+yields!(atomic_rec, "try .. catch 0", json!(null));
+yields!(atomic_id, "try . catch 0", json!(null));
+yields!(atomic_key1, "{key: 0} | try .key catch 1", 0);
+yields!(atomic_key2, r#"{key: 0} | try . "key" catch 1"#, 0);
+yields!(atomic_key3, r#"def @f: .; {key: 0} | try .@f"key" catch 1"#, 0);
+yields!(atomic_num, "try 0 catch 1", 0);
+yields!(atomic_block, "try (1 + 1) catch 0", 2);
+yields!(atomic_path, "try [1][0] catch 0", 1);
+yields!(atomic_opt, "def x: 0; try x? catch 1", 0);
+
 yields!(neg_arr_iter1, "[-[][]]", json!([]));
 yields!(neg_arr_iter2, "try (-[])[] catch 0", 0);
 
@@ -164,6 +185,8 @@ yields!(
     "{a: 1, b: 2} | {a, c: 3}",
     json!({"a": 1, "c": 3})
 );
+yields!(obj_var, r#""x" as $k | {$k}"#, json!({"k": "x"}));
+yields!(obj_var_val, r#""x" as $k | {$k: 0}"#, json!({"x": 0}));
 yields!(
     obj_multi_keys,
     r#"[{("a", "b"): 1}]"#,

jaq-parse/src/lib.rs

@@ -12,14 +12,12 @@ mod prec_climb;
 mod string;
 mod token;
 
-use jaq_syn as syn;
-
 pub use def::{defs, main};
 use token::{Delim, Token};
 
 use alloc::{string::String, string::ToString, vec::Vec};
 use chumsky::prelude::*;
-use syn::Spanned;
+use jaq_syn::Spanned;
 
 /// Lex/parse error.
 pub type Error = Simple<String>;

jaq-play/Cargo.toml

@@ -17,13 +17,11 @@ crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 jaq-syn       = { version = "1.1.0", path = "../jaq-syn" }
-jaq-parse     = { version = "1.0.0", path = "../jaq-parse" }
 jaq-interpret = { version = "1.2.0", path = "../jaq-interpret" }
 jaq-core      = { version = "1.2.0", path = "../jaq-core" }
 jaq-std       = { version = "1.2.0", path = "../jaq-std" }
 aho-corasick = "1.1.2"
 codesnake = { version = "0.1" }
-chumsky = { version = "0.9.0", default-features = false }
 hifijson = "0.2"
 log = "0.4.17"
 unicode-width = "0.1.13"